Dense.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_VIEWS_SUBMATRIX_DENSE_H_
36 #define _BLAZE_MATH_VIEWS_SUBMATRIX_DENSE_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <algorithm>
44 #include <iterator>
45 #include <blaze/math/Aliases.h>
57 #include <blaze/math/Exception.h>
61 #include <blaze/math/shims/Clear.h>
63 #include <blaze/math/shims/Reset.h>
64 #include <blaze/math/SIMD.h>
90 #include <blaze/math/views/Check.h>
93 #include <blaze/system/Blocking.h>
94 #include <blaze/system/CacheSize.h>
95 #include <blaze/system/Inline.h>
101 #include <blaze/util/Assert.h>
105 #include <blaze/util/DisableIf.h>
106 #include <blaze/util/EnableIf.h>
107 #include <blaze/util/mpl/And.h>
108 #include <blaze/util/mpl/If.h>
109 #include <blaze/util/mpl/Not.h>
110 #include <blaze/util/mpl/Or.h>
111 #include <blaze/util/Template.h>
112 #include <blaze/util/TypeList.h>
113 #include <blaze/util/Types.h>
116 #include <blaze/util/Unused.h>
117 
118 
119 namespace blaze {
120 
121 //=================================================================================================
122 //
123 // CLASS TEMPLATE SPECIALIZATION FOR UNALIGNED ROW-MAJOR DENSE SUBMATRICES
124 //
125 //=================================================================================================
126 
127 //*************************************************************************************************
135 template< typename MT // Type of the dense matrix
136  , size_t... CSAs > // Compile time submatrix arguments
137 class Submatrix<MT,unaligned,false,true,CSAs...>
138  : public View< DenseMatrix< Submatrix<MT,unaligned,false,true,CSAs...>, false > >
139  , private SubmatrixData<CSAs...>
140 {
141  private:
142  //**Type definitions****************************************************************************
143  using DataType = SubmatrixData<CSAs...>;
144  using Operand = If_< IsExpression<MT>, MT, MT& >;
145  //**********************************************************************************************
146 
147  public:
148  //**Type definitions****************************************************************************
150  using This = Submatrix<MT,unaligned,false,true,CSAs...>;
151 
152  using BaseType = DenseMatrix<This,false>;
153  using ViewedType = MT;
154  using ResultType = SubmatrixTrait_<MT,CSAs...>;
155  using OppositeType = OppositeType_<ResultType>;
156  using TransposeType = TransposeType_<ResultType>;
157  using ElementType = ElementType_<MT>;
158  using SIMDType = SIMDTrait_<ElementType>;
159  using ReturnType = ReturnType_<MT>;
160  using CompositeType = const Submatrix&;
161 
163  using ConstReference = ConstReference_<MT>;
164 
166  using Reference = If_< IsConst<MT>, ConstReference, Reference_<MT> >;
167 
169  using ConstPointer = ConstPointer_<MT>;
170 
172  using Pointer = If_< Or< IsConst<MT>, Not< HasMutableDataAccess<MT> > >, ConstPointer, Pointer_<MT> >;
173  //**********************************************************************************************
174 
175  //**SubmatrixIterator class definition**********************************************************
178  template< typename IteratorType > // Type of the dense matrix iterator
179  class SubmatrixIterator
180  {
181  public:
182  //**Type definitions*************************************************************************
184  using IteratorCategory = typename std::iterator_traits<IteratorType>::iterator_category;
185 
187  using ValueType = typename std::iterator_traits<IteratorType>::value_type;
188 
190  using PointerType = typename std::iterator_traits<IteratorType>::pointer;
191 
193  using ReferenceType = typename std::iterator_traits<IteratorType>::reference;
194 
196  using DifferenceType = typename std::iterator_traits<IteratorType>::difference_type;
197 
198  // STL iterator requirements
199  using iterator_category = IteratorCategory;
200  using value_type = ValueType;
201  using pointer = PointerType;
202  using reference = ReferenceType;
203  using difference_type = DifferenceType;
204  //*******************************************************************************************
205 
206  //**Constructor******************************************************************************
209  inline SubmatrixIterator()
210  : iterator_ ( ) // Iterator to the current submatrix element
211  , isAligned_( false ) // Memory alignment flag
212  {}
213  //*******************************************************************************************
214 
215  //**Constructor******************************************************************************
221  inline SubmatrixIterator( IteratorType iterator, bool isMemoryAligned )
222  : iterator_ ( iterator ) // Iterator to the current submatrix element
223  , isAligned_( isMemoryAligned ) // Memory alignment flag
224  {}
225  //*******************************************************************************************
226 
227  //**Constructor******************************************************************************
232  template< typename IteratorType2 >
233  inline SubmatrixIterator( const SubmatrixIterator<IteratorType2>& it )
234  : iterator_ ( it.base() ) // Iterator to the current submatrix element
235  , isAligned_( it.isAligned() ) // Memory alignment flag
236  {}
237  //*******************************************************************************************
238 
239  //**Addition assignment operator*************************************************************
245  inline SubmatrixIterator& operator+=( size_t inc ) {
246  iterator_ += inc;
247  return *this;
248  }
249  //*******************************************************************************************
250 
251  //**Subtraction assignment operator**********************************************************
257  inline SubmatrixIterator& operator-=( size_t dec ) {
258  iterator_ -= dec;
259  return *this;
260  }
261  //*******************************************************************************************
262 
263  //**Prefix increment operator****************************************************************
268  inline SubmatrixIterator& operator++() {
269  ++iterator_;
270  return *this;
271  }
272  //*******************************************************************************************
273 
274  //**Postfix increment operator***************************************************************
279  inline const SubmatrixIterator operator++( int ) {
280  return SubmatrixIterator( iterator_++, isAligned_ );
281  }
282  //*******************************************************************************************
283 
284  //**Prefix decrement operator****************************************************************
289  inline SubmatrixIterator& operator--() {
290  --iterator_;
291  return *this;
292  }
293  //*******************************************************************************************
294 
295  //**Postfix decrement operator***************************************************************
300  inline const SubmatrixIterator operator--( int ) {
301  return SubmatrixIterator( iterator_--, isAligned_ );
302  }
303  //*******************************************************************************************
304 
305  //**Element access operator******************************************************************
310  inline ReferenceType operator*() const {
311  return *iterator_;
312  }
313  //*******************************************************************************************
314 
315  //**Element access operator******************************************************************
320  inline IteratorType operator->() const {
321  return iterator_;
322  }
323  //*******************************************************************************************
324 
325  //**Load function****************************************************************************
335  inline SIMDType load() const noexcept {
336  if( isAligned_ )
337  return loada();
338  else
339  return loadu();
340  }
341  //*******************************************************************************************
342 
343  //**Loada function***************************************************************************
353  inline SIMDType loada() const noexcept {
354  return iterator_.loada();
355  }
356  //*******************************************************************************************
357 
358  //**Loadu function***************************************************************************
368  inline SIMDType loadu() const noexcept {
369  return iterator_.loadu();
370  }
371  //*******************************************************************************************
372 
373  //**Store function***************************************************************************
384  inline void store( const SIMDType& value ) const {
385  storeu( value );
386  }
387  //*******************************************************************************************
388 
389  //**Storea function**************************************************************************
400  inline void storea( const SIMDType& value ) const {
401  iterator_.storea( value );
402  }
403  //*******************************************************************************************
404 
405  //**Storeu function**************************************************************************
416  inline void storeu( const SIMDType& value ) const {
417  if( isAligned_ ) {
418  iterator_.storea( value );
419  }
420  else {
421  iterator_.storeu( value );
422  }
423  }
424  //*******************************************************************************************
425 
426  //**Stream function**************************************************************************
437  inline void stream( const SIMDType& value ) const {
438  iterator_.stream( value );
439  }
440  //*******************************************************************************************
441 
442  //**Equality operator************************************************************************
448  inline bool operator==( const SubmatrixIterator& rhs ) const {
449  return iterator_ == rhs.iterator_;
450  }
451  //*******************************************************************************************
452 
453  //**Inequality operator**********************************************************************
459  inline bool operator!=( const SubmatrixIterator& rhs ) const {
460  return iterator_ != rhs.iterator_;
461  }
462  //*******************************************************************************************
463 
464  //**Less-than operator***********************************************************************
470  inline bool operator<( const SubmatrixIterator& rhs ) const {
471  return iterator_ < rhs.iterator_;
472  }
473  //*******************************************************************************************
474 
475  //**Greater-than operator********************************************************************
481  inline bool operator>( const SubmatrixIterator& rhs ) const {
482  return iterator_ > rhs.iterator_;
483  }
484  //*******************************************************************************************
485 
486  //**Less-or-equal-than operator**************************************************************
492  inline bool operator<=( const SubmatrixIterator& rhs ) const {
493  return iterator_ <= rhs.iterator_;
494  }
495  //*******************************************************************************************
496 
497  //**Greater-or-equal-than operator***********************************************************
503  inline bool operator>=( const SubmatrixIterator& rhs ) const {
504  return iterator_ >= rhs.iterator_;
505  }
506  //*******************************************************************************************
507 
508  //**Subtraction operator*********************************************************************
514  inline DifferenceType operator-( const SubmatrixIterator& rhs ) const {
515  return iterator_ - rhs.iterator_;
516  }
517  //*******************************************************************************************
518 
519  //**Addition operator************************************************************************
526  friend inline const SubmatrixIterator operator+( const SubmatrixIterator& it, size_t inc ) {
527  return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
528  }
529  //*******************************************************************************************
530 
531  //**Addition operator************************************************************************
538  friend inline const SubmatrixIterator operator+( size_t inc, const SubmatrixIterator& it ) {
539  return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
540  }
541  //*******************************************************************************************
542 
543  //**Subtraction operator*********************************************************************
550  friend inline const SubmatrixIterator operator-( const SubmatrixIterator& it, size_t dec ) {
551  return SubmatrixIterator( it.iterator_ - dec, it.isAligned_ );
552  }
553  //*******************************************************************************************
554 
555  //**Base function****************************************************************************
560  inline IteratorType base() const {
561  return iterator_;
562  }
563  //*******************************************************************************************
564 
565  //**IsAligned function***********************************************************************
570  inline bool isAligned() const noexcept {
571  return isAligned_;
572  }
573  //*******************************************************************************************
574 
575  private:
576  //**Member variables*************************************************************************
577  IteratorType iterator_;
578  bool isAligned_;
579  //*******************************************************************************************
580  };
581  //**********************************************************************************************
582 
583  //**Type definitions****************************************************************************
585  using ConstIterator = SubmatrixIterator< ConstIterator_<MT> >;
586 
588  using Iterator = If_< IsConst<MT>, ConstIterator, SubmatrixIterator< Iterator_<MT> > >;
589  //**********************************************************************************************
590 
591  //**Compilation flags***************************************************************************
593  enum : bool { simdEnabled = MT::simdEnabled };
594 
596  enum : bool { smpAssignable = MT::smpAssignable };
597  //**********************************************************************************************
598 
599  //**Constructors********************************************************************************
602  template< typename... RSAs >
603  explicit inline Submatrix( MT& matrix, RSAs... args );
604  // No explicitly declared copy constructor.
606  //**********************************************************************************************
607 
608  //**Destructor**********************************************************************************
609  // No explicitly declared destructor.
610  //**********************************************************************************************
611 
612  //**Data access functions***********************************************************************
615  inline Reference operator()( size_t i, size_t j );
616  inline ConstReference operator()( size_t i, size_t j ) const;
617  inline Reference at( size_t i, size_t j );
618  inline ConstReference at( size_t i, size_t j ) const;
619  inline Pointer data () noexcept;
620  inline ConstPointer data () const noexcept;
621  inline Pointer data ( size_t i ) noexcept;
622  inline ConstPointer data ( size_t i ) const noexcept;
623  inline Iterator begin ( size_t i );
624  inline ConstIterator begin ( size_t i ) const;
625  inline ConstIterator cbegin( size_t i ) const;
626  inline Iterator end ( size_t i );
627  inline ConstIterator end ( size_t i ) const;
628  inline ConstIterator cend ( size_t i ) const;
630  //**********************************************************************************************
631 
632  //**Assignment operators************************************************************************
635  inline Submatrix& operator=( const ElementType& rhs );
636  inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
637  inline Submatrix& operator=( const Submatrix& rhs );
638 
639  template< typename MT2, bool SO2 >
640  inline Submatrix& operator=( const Matrix<MT2,SO2>& rhs );
641 
642  template< typename MT2, bool SO2 >
643  inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
644  operator+=( const Matrix<MT2,SO2>& rhs );
645 
646  template< typename MT2, bool SO2 >
647  inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
648  operator+=( const Matrix<MT2,SO2>& rhs );
649 
650  template< typename MT2, bool SO2 >
651  inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
652  operator-=( const Matrix<MT2,SO2>& rhs );
653 
654  template< typename MT2, bool SO2 >
655  inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
656  operator-=( const Matrix<MT2,SO2>& rhs );
657 
658  template< typename MT2, bool SO2 >
659  inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
660  operator%=( const Matrix<MT2,SO2>& rhs );
661 
662  template< typename MT2, bool SO2 >
663  inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
664  operator%=( const Matrix<MT2,SO2>& rhs );
666  //**********************************************************************************************
667 
668  //**Utility functions***************************************************************************
671  using DataType::row;
672  using DataType::column;
673  using DataType::rows;
674  using DataType::columns;
675 
676  inline MT& operand() noexcept;
677  inline const MT& operand() const noexcept;
678 
679  inline size_t spacing() const noexcept;
680  inline size_t capacity() const noexcept;
681  inline size_t capacity( size_t i ) const noexcept;
682  inline size_t nonZeros() const;
683  inline size_t nonZeros( size_t i ) const;
684  inline void reset();
685  inline void reset( size_t i );
687  //**********************************************************************************************
688 
689  //**Numeric functions***************************************************************************
692  inline Submatrix& transpose();
693  inline Submatrix& ctranspose();
694 
695  template< typename Other > inline Submatrix& scale( const Other& scalar );
697  //**********************************************************************************************
698 
699  private:
700  //**********************************************************************************************
702  template< typename MT2 >
703  struct VectorizedAssign {
704  enum : bool { value = useOptimizedKernels &&
705  simdEnabled && MT2::simdEnabled &&
706  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value };
707  };
708  //**********************************************************************************************
709 
710  //**********************************************************************************************
712  template< typename MT2 >
713  struct VectorizedAddAssign {
714  enum : bool { value = useOptimizedKernels &&
715  simdEnabled && MT2::simdEnabled &&
716  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value &&
717  HasSIMDAdd< ElementType, ElementType_<MT2> >::value &&
718  !IsDiagonal<MT2>::value };
719  };
720  //**********************************************************************************************
721 
722  //**********************************************************************************************
724  template< typename MT2 >
725  struct VectorizedSubAssign {
726  enum : bool { value = useOptimizedKernels &&
727  simdEnabled && MT2::simdEnabled &&
728  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value &&
729  HasSIMDSub< ElementType, ElementType_<MT2> >::value &&
730  !IsDiagonal<MT2>::value };
731  };
732  //**********************************************************************************************
733 
734  //**********************************************************************************************
736  template< typename MT2 >
737  struct VectorizedSchurAssign {
738  enum : bool { value = useOptimizedKernels &&
739  simdEnabled && MT2::simdEnabled &&
740  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value &&
741  HasSIMDMult< ElementType, ElementType_<MT2> >::value };
742  };
743  //**********************************************************************************************
744 
745  //**SIMD properties*****************************************************************************
747  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
748  //**********************************************************************************************
749 
750  public:
751  //**Expression template evaluation functions****************************************************
754  template< typename Other >
755  inline bool canAlias( const Other* alias ) const noexcept;
756 
757  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
758  inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
759 
760  template< typename Other >
761  inline bool isAliased( const Other* alias ) const noexcept;
762 
763  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
764  inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
765 
766  inline bool isAligned () const noexcept;
767  inline bool canSMPAssign() const noexcept;
768 
769  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
770  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
771  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
772 
773  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
774  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
775  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
776  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
777 
778  template< typename MT2 >
779  inline DisableIf_< VectorizedAssign<MT2> > assign( const DenseMatrix<MT2,false>& rhs );
780 
781  template< typename MT2 >
782  inline EnableIf_< VectorizedAssign<MT2> > assign( const DenseMatrix<MT2,false>& rhs );
783 
784  template< typename MT2 > inline void assign( const DenseMatrix<MT2,true>& rhs );
785  template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
786  template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
787 
788  template< typename MT2 >
789  inline DisableIf_< VectorizedAddAssign<MT2> > addAssign( const DenseMatrix<MT2,false>& rhs );
790 
791  template< typename MT2 >
792  inline EnableIf_< VectorizedAddAssign<MT2> > addAssign( const DenseMatrix<MT2,false>& rhs );
793 
794  template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,true>& rhs );
795  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
796  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
797 
798  template< typename MT2 >
799  inline DisableIf_< VectorizedSubAssign<MT2> > subAssign( const DenseMatrix<MT2,false>& rhs );
800 
801  template< typename MT2 >
802  inline EnableIf_< VectorizedSubAssign<MT2> > subAssign( const DenseMatrix<MT2,false>& rhs );
803 
804  template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,true>& rhs );
805  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
806  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
807 
808  template< typename MT2 >
809  inline DisableIf_< VectorizedSchurAssign<MT2> > schurAssign( const DenseMatrix<MT2,false>& rhs );
810 
811  template< typename MT2 >
812  inline EnableIf_< VectorizedSchurAssign<MT2> > schurAssign( const DenseMatrix<MT2,false>& rhs );
813 
814  template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,true>& rhs );
815  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
816  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
818  //**********************************************************************************************
819 
820  private:
821  //**Utility functions***************************************************************************
824  inline bool hasOverlap() const noexcept;
826  //**********************************************************************************************
827 
828  //**Member variables****************************************************************************
831  Operand matrix_;
832  const bool isAligned_;
833 
840  //**********************************************************************************************
841 
842  //**Friend declarations*************************************************************************
843  template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
844  //**********************************************************************************************
845 
846  //**Compile time checks*************************************************************************
854  //**********************************************************************************************
855 };
857 //*************************************************************************************************
858 
859 
860 
861 
862 //=================================================================================================
863 //
864 // CONSTRUCTORS
865 //
866 //=================================================================================================
867 
868 //*************************************************************************************************
881 template< typename MT // Type of the dense matrix
882  , size_t... CSAs > // Compile time submatrix arguments
883 template< typename... RSAs > // Runtime submatrix arguments
884 inline Submatrix<MT,unaligned,false,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
885  : DataType ( args... ) // Base class initialization
886  , matrix_ ( matrix ) // The matrix containing the submatrix
887  , isAligned_( simdEnabled && matrix.data() != nullptr && checkAlignment( data() ) &&
888  ( rows() < 2UL || ( matrix.spacing() & size_t(-SIMDSIZE) ) == 0UL ) )
889 {
890  if( !Contains< TypeList<RSAs...>, Unchecked >::value ) {
891  if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
892  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
893  }
894  }
895  else {
896  BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
897  BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
898  }
899 }
901 //*************************************************************************************************
902 
903 
904 
905 
906 //=================================================================================================
907 //
908 // DATA ACCESS FUNCTIONS
909 //
910 //=================================================================================================
911 
912 //*************************************************************************************************
923 template< typename MT // Type of the dense matrix
924  , size_t... CSAs > // Compile time submatrix arguments
925 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Reference
926  Submatrix<MT,unaligned,false,true,CSAs...>::operator()( size_t i, size_t j )
927 {
928  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
929  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
930 
931  return matrix_(row()+i,column()+j);
932 }
934 //*************************************************************************************************
935 
936 
937 //*************************************************************************************************
948 template< typename MT // Type of the dense matrix
949  , size_t... CSAs > // Compile time submatrix arguments
950 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstReference
951  Submatrix<MT,unaligned,false,true,CSAs...>::operator()( size_t i, size_t j ) const
952 {
953  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
954  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
955 
956  return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
957 }
959 //*************************************************************************************************
960 
961 
962 //*************************************************************************************************
974 template< typename MT // Type of the dense matrix
975  , size_t... CSAs > // Compile time submatrix arguments
976 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Reference
977  Submatrix<MT,unaligned,false,true,CSAs...>::at( size_t i, size_t j )
978 {
979  if( i >= rows() ) {
980  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
981  }
982  if( j >= columns() ) {
983  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
984  }
985  return (*this)(i,j);
986 }
988 //*************************************************************************************************
989 
990 
991 //*************************************************************************************************
1003 template< typename MT // Type of the dense matrix
1004  , size_t... CSAs > // Compile time submatrix arguments
1005 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstReference
1006  Submatrix<MT,unaligned,false,true,CSAs...>::at( size_t i, size_t j ) const
1007 {
1008  if( i >= rows() ) {
1009  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
1010  }
1011  if( j >= columns() ) {
1012  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
1013  }
1014  return (*this)(i,j);
1015 }
1017 //*************************************************************************************************
1018 
1019 
1020 //*************************************************************************************************
1030 template< typename MT // Type of the dense matrix
1031  , size_t... CSAs > // Compile time submatrix arguments
1032 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Pointer
1034 {
1035  return matrix_.data() + row()*spacing() + column();
1036 }
1038 //*************************************************************************************************
1039 
1040 
1041 //*************************************************************************************************
1051 template< typename MT // Type of the dense matrix
1052  , size_t... CSAs > // Compile time submatrix arguments
1053 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstPointer
1055 {
1056  return matrix_.data() + row()*spacing() + column();
1057 }
1059 //*************************************************************************************************
1060 
1061 
1062 //*************************************************************************************************
1071 template< typename MT // Type of the dense matrix
1072  , size_t... CSAs > // Compile time submatrix arguments
1073 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Pointer
1075 {
1076  return matrix_.data() + (row()+i)*spacing() + column();
1077 }
1079 //*************************************************************************************************
1080 
1081 
1082 //*************************************************************************************************
1091 template< typename MT // Type of the dense matrix
1092  , size_t... CSAs > // Compile time submatrix arguments
1093 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstPointer
1094  Submatrix<MT,unaligned,false,true,CSAs...>::data( size_t i ) const noexcept
1095 {
1096  return matrix_.data() + (row()+i)*spacing() + column();
1097 }
1099 //*************************************************************************************************
1100 
1101 
1102 //*************************************************************************************************
1114 template< typename MT // Type of the dense matrix
1115  , size_t... CSAs > // Compile time submatrix arguments
1116 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Iterator
1118 {
1119  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1120  return Iterator( matrix_.begin( row() + i ) + column(), isAligned_ );
1121 }
1123 //*************************************************************************************************
1124 
1125 
1126 //*************************************************************************************************
1138 template< typename MT // Type of the dense matrix
1139  , size_t... CSAs > // Compile time submatrix arguments
1140 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1142 {
1143  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1144  return ConstIterator( matrix_.cbegin( row() + i ) + column(), isAligned_ );
1145 }
1147 //*************************************************************************************************
1148 
1149 
1150 //*************************************************************************************************
1162 template< typename MT // Type of the dense matrix
1163  , size_t... CSAs > // Compile time submatrix arguments
1164 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1166 {
1167  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1168  return ConstIterator( matrix_.cbegin( row() + i ) + column(), isAligned_ );
1169 }
1171 //*************************************************************************************************
1172 
1173 
1174 //*************************************************************************************************
1186 template< typename MT // Type of the dense matrix
1187  , size_t... CSAs > // Compile time submatrix arguments
1188 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Iterator
1190 {
1191  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1192  return Iterator( matrix_.begin( row() + i ) + column() + columns(), isAligned_ );
1193 }
1195 //*************************************************************************************************
1196 
1197 
1198 //*************************************************************************************************
1210 template< typename MT // Type of the dense matrix
1211  , size_t... CSAs > // Compile time submatrix arguments
1212 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1214 {
1215  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1216  return ConstIterator( matrix_.cbegin( row() + i ) + column() + columns(), isAligned_ );
1217 }
1219 //*************************************************************************************************
1220 
1221 
1222 //*************************************************************************************************
1234 template< typename MT // Type of the dense matrix
1235  , size_t... CSAs > // Compile time submatrix arguments
1236 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1238 {
1239  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1240  return ConstIterator( matrix_.cbegin( row() + i ) + column() + columns(), isAligned_ );
1241 }
1243 //*************************************************************************************************
1244 
1245 
1246 
1247 
1248 //=================================================================================================
1249 //
1250 // ASSIGNMENT OPERATORS
1251 //
1252 //=================================================================================================
1253 
1254 //*************************************************************************************************
1265 template< typename MT // Type of the dense matrix
1266  , size_t... CSAs > // Compile time submatrix arguments
1267 inline Submatrix<MT,unaligned,false,true,CSAs...>&
1268  Submatrix<MT,unaligned,false,true,CSAs...>::operator=( const ElementType& rhs )
1269 {
1270  const size_t iend( row() + rows() );
1271  decltype(auto) left( derestrict( matrix_ ) );
1272 
1273  for( size_t i=row(); i<iend; ++i )
1274  {
1275  const size_t jbegin( ( IsUpper<MT>::value )
1276  ?( ( IsUniUpper<MT>::value || IsStrictlyUpper<MT>::value )
1277  ?( max( i+1UL, column() ) )
1278  :( max( i, column() ) ) )
1279  :( column() ) );
1280  const size_t jend ( ( IsLower<MT>::value )
1281  ?( ( IsUniLower<MT>::value || IsStrictlyLower<MT>::value )
1282  ?( min( i, column()+columns() ) )
1283  :( min( i+1UL, column()+columns() ) ) )
1284  :( column()+columns() ) );
1285 
1286  for( size_t j=jbegin; j<jend; ++j ) {
1287  if( !IsRestricted<MT>::value || IsTriangular<MT>::value || trySet( matrix_, i, j, rhs ) )
1288  left(i,j) = rhs;
1289  }
1290  }
1291 
1292  return *this;
1293 }
1295 //*************************************************************************************************
1296 
1297 
1298 //*************************************************************************************************
1314 template< typename MT // Type of the dense matrix
1315  , size_t... CSAs > // Compile time submatrix arguments
1316 inline Submatrix<MT,unaligned,false,true,CSAs...>&
1317  Submatrix<MT,unaligned,false,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
1318 {
1319  if( list.size() != rows() ) {
1320  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
1321  }
1322 
1323  if( IsRestricted<MT>::value ) {
1324  const InitializerMatrix<ElementType> tmp( list, columns() );
1325  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1326  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1327  }
1328  }
1329 
1330  decltype(auto) left( derestrict( *this ) );
1331  size_t i( 0UL );
1332 
1333  for( const auto& rowList : list ) {
1334  std::fill( std::copy( rowList.begin(), rowList.end(), left.begin(i) ), left.end(i), ElementType() );
1335  ++i;
1336  }
1337 
1338  return *this;
1339 }
1341 //*************************************************************************************************
1342 
1343 
1344 //*************************************************************************************************
1359 template< typename MT // Type of the dense matrix
1360  , size_t... CSAs > // Compile time submatrix arguments
1361 inline Submatrix<MT,unaligned,false,true,CSAs...>&
1362  Submatrix<MT,unaligned,false,true,CSAs...>::operator=( const Submatrix& rhs )
1363 {
1366 
1367  if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
1368  return *this;
1369 
1370  if( rows() != rhs.rows() || columns() != rhs.columns() ) {
1371  BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
1372  }
1373 
1374  if( !tryAssign( matrix_, rhs, row(), column() ) ) {
1375  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1376  }
1377 
1378  decltype(auto) left( derestrict( *this ) );
1379 
1380  if( rhs.canAlias( &matrix_ ) ) {
1381  const ResultType tmp( rhs );
1382  smpAssign( left, tmp );
1383  }
1384  else {
1385  smpAssign( left, rhs );
1386  }
1387 
1388  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1389 
1390  return *this;
1391 }
1393 //*************************************************************************************************
1394 
1395 
1396 //*************************************************************************************************
1411 template< typename MT // Type of the dense matrix
1412  , size_t... CSAs > // Compile time submatrix arguments
1413 template< typename MT2 // Type of the right-hand side matrix
1414  , bool SO2 > // Storage order of the right-hand side matrix
1415 inline Submatrix<MT,unaligned,false,true,CSAs...>&
1416  Submatrix<MT,unaligned,false,true,CSAs...>::operator=( const Matrix<MT2,SO2>& rhs )
1417 {
1419 
1420  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1421  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1422  }
1423 
1424  using Right = If_< IsRestricted<MT>, CompositeType_<MT2>, const MT2& >;
1425  Right right( ~rhs );
1426 
1427  if( !tryAssign( matrix_, right, row(), column() ) ) {
1428  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1429  }
1430 
1431  decltype(auto) left( derestrict( *this ) );
1432 
1433  if( IsReference<Right>::value && right.canAlias( &matrix_ ) ) {
1434  const ResultType_<MT2> tmp( right );
1435  if( IsSparseMatrix<MT2>::value )
1436  reset();
1437  smpAssign( left, tmp );
1438  }
1439  else {
1440  if( IsSparseMatrix<MT2>::value )
1441  reset();
1442  smpAssign( left, right );
1443  }
1444 
1445  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1446 
1447  return *this;
1448 }
1450 //*************************************************************************************************
1451 
1452 
1453 //*************************************************************************************************
1467 template< typename MT // Type of the dense matrix
1468  , size_t... CSAs > // Compile time submatrix arguments
1469 template< typename MT2 // Type of the right-hand side matrix
1470  , bool SO2 > // Storage order of the right-hand side matrix
1471 inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,unaligned,false,true,CSAs...>& >
1472  Submatrix<MT,unaligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO2>& rhs )
1473 {
1477 
1478  using AddType = AddTrait_< ResultType, ResultType_<MT2> >;
1479 
1482 
1483  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1484  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1485  }
1486 
1487  if( !tryAddAssign( matrix_, ~rhs, row(), column() ) ) {
1488  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1489  }
1490 
1491  decltype(auto) left( derestrict( *this ) );
1492 
1493  if( ( ( IsSymmetric<MT>::value || IsHermitian<MT>::value ) && hasOverlap() ) ||
1494  (~rhs).canAlias( &matrix_ ) ) {
1495  const AddType tmp( *this + (~rhs) );
1496  smpAssign( left, tmp );
1497  }
1498  else {
1499  smpAddAssign( left, ~rhs );
1500  }
1501 
1502  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1503 
1504  return *this;
1505 }
1507 //*************************************************************************************************
1508 
1509 
1510 //*************************************************************************************************
1524 template< typename MT // Type of the dense matrix
1525  , size_t... CSAs > // Compile time submatrix arguments
1526 template< typename MT2 // Type of the right-hand side matrix
1527  , bool SO2 > // Storage order of the right-hand side matrix
1528 inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,unaligned,false,true,CSAs...>& >
1529  Submatrix<MT,unaligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO2>& rhs )
1530 {
1534 
1535  using AddType = AddTrait_< ResultType, ResultType_<MT2> >;
1536 
1539 
1540  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1541  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1542  }
1543 
1544  const AddType tmp( *this + (~rhs) );
1545 
1546  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1547  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1548  }
1549 
1550  decltype(auto) left( derestrict( *this ) );
1551 
1552  smpAssign( left, tmp );
1553 
1554  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1555 
1556  return *this;
1557 }
1559 //*************************************************************************************************
1560 
1561 
1562 //*************************************************************************************************
1576 template< typename MT // Type of the dense matrix
1577  , size_t... CSAs > // Compile time submatrix arguments
1578 template< typename MT2 // Type of the right-hand side matrix
1579  , bool SO2 > // Storage order of the right-hand side matrix
1580 inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,unaligned,false,true,CSAs...>& >
1581  Submatrix<MT,unaligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO2>& rhs )
1582 {
1586 
1587  using SubType = SubTrait_< ResultType, ResultType_<MT2> >;
1588 
1591 
1592  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1593  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1594  }
1595 
1596  if( !trySubAssign( matrix_, ~rhs, row(), column() ) ) {
1597  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1598  }
1599 
1600  decltype(auto) left( derestrict( *this ) );
1601 
1602  if( ( ( IsSymmetric<MT>::value || IsHermitian<MT>::value ) && hasOverlap() ) ||
1603  (~rhs).canAlias( &matrix_ ) ) {
1604  const SubType tmp( *this - (~rhs ) );
1605  smpAssign( left, tmp );
1606  }
1607  else {
1608  smpSubAssign( left, ~rhs );
1609  }
1610 
1611  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1612 
1613  return *this;
1614 }
1616 //*************************************************************************************************
1617 
1618 
1619 //*************************************************************************************************
1633 template< typename MT // Type of the dense matrix
1634  , size_t... CSAs > // Compile time submatrix arguments
1635 template< typename MT2 // Type of the right-hand side matrix
1636  , bool SO2 > // Storage order of the right-hand side matrix
1637 inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,unaligned,false,true,CSAs...>& >
1638  Submatrix<MT,unaligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO2>& rhs )
1639 {
1643 
1644  using SubType = SubTrait_< ResultType, ResultType_<MT2> >;
1645 
1648 
1649  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1650  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1651  }
1652 
1653  const SubType tmp( *this - (~rhs) );
1654 
1655  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1656  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1657  }
1658 
1659  decltype(auto) left( derestrict( *this ) );
1660 
1661  smpAssign( left, tmp );
1662 
1663  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1664 
1665  return *this;
1666 }
1668 //*************************************************************************************************
1669 
1670 
1671 //*************************************************************************************************
1685 template< typename MT // Type of the dense matrix
1686  , size_t... CSAs > // Compile time submatrix arguments
1687 template< typename MT2 // Type of the right-hand side matrix
1688  , bool SO2 > // Storage order of the right-hand side matrix
1689 inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,unaligned,false,true,CSAs...>& >
1690  Submatrix<MT,unaligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO2>& rhs )
1691 {
1695 
1696  using SchurType = SchurTrait_< ResultType, ResultType_<MT2> >;
1697 
1699 
1700  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1701  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1702  }
1703 
1704  if( !trySchurAssign( matrix_, ~rhs, row(), column() ) ) {
1705  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1706  }
1707 
1708  decltype(auto) left( derestrict( *this ) );
1709 
1710  if( ( ( IsSymmetric<MT>::value || IsHermitian<MT>::value ) && hasOverlap() ) ||
1711  (~rhs).canAlias( &matrix_ ) ) {
1712  const SchurType tmp( *this % (~rhs) );
1713  if( IsSparseMatrix<SchurType>::value )
1714  reset();
1715  smpAssign( left, tmp );
1716  }
1717  else {
1718  smpSchurAssign( left, ~rhs );
1719  }
1720 
1721  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1722 
1723  return *this;
1724 }
1726 //*************************************************************************************************
1727 
1728 
1729 //*************************************************************************************************
1743 template< typename MT // Type of the dense matrix
1744  , size_t... CSAs > // Compile time submatrix arguments
1745 template< typename MT2 // Type of the right-hand side matrix
1746  , bool SO2 > // Storage order of the right-hand side matrix
1747 inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,unaligned,false,true,CSAs...>& >
1748  Submatrix<MT,unaligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO2>& rhs )
1749 {
1753 
1754  using SchurType = SchurTrait_< ResultType, ResultType_<MT2> >;
1755 
1757 
1758  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1759  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1760  }
1761 
1762  const SchurType tmp( *this % (~rhs) );
1763 
1764  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1765  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1766  }
1767 
1768  decltype(auto) left( derestrict( *this ) );
1769 
1770  if( IsSparseMatrix<SchurType>::value ) {
1771  reset();
1772  }
1773 
1774  smpAssign( left, tmp );
1775 
1776  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1777 
1778  return *this;
1779 }
1781 //*************************************************************************************************
1782 
1783 
1784 
1785 
1786 //=================================================================================================
1787 //
1788 // UTILITY FUNCTIONS
1789 //
1790 //=================================================================================================
1791 
1792 //*************************************************************************************************
1798 template< typename MT // Type of the dense matrix
1799  , size_t... CSAs > // Compile time submatrix arguments
1800 inline MT& Submatrix<MT,unaligned,false,true,CSAs...>::operand() noexcept
1801 {
1802  return matrix_;
1803 }
1805 //*************************************************************************************************
1806 
1807 
1808 //*************************************************************************************************
1814 template< typename MT // Type of the dense matrix
1815  , size_t... CSAs > // Compile time submatrix arguments
1816 inline const MT& Submatrix<MT,unaligned,false,true,CSAs...>::operand() const noexcept
1817 {
1818  return matrix_;
1819 }
1821 //*************************************************************************************************
1822 
1823 
1824 //*************************************************************************************************
1835 template< typename MT // Type of the dense matrix
1836  , size_t... CSAs > // Compile time submatrix arguments
1837 inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::spacing() const noexcept
1838 {
1839  return matrix_.spacing();
1840 }
1842 //*************************************************************************************************
1843 
1844 
1845 //*************************************************************************************************
1851 template< typename MT // Type of the dense matrix
1852  , size_t... CSAs > // Compile time submatrix arguments
1853 inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::capacity() const noexcept
1854 {
1855  return rows() * columns();
1856 }
1858 //*************************************************************************************************
1859 
1860 
1861 //*************************************************************************************************
1873 template< typename MT // Type of the dense matrix
1874  , size_t... CSAs > // Compile time submatrix arguments
1875 inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::capacity( size_t i ) const noexcept
1876 {
1877  UNUSED_PARAMETER( i );
1878 
1879  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1880 
1881  return columns();
1882 }
1884 //*************************************************************************************************
1885 
1886 
1887 //*************************************************************************************************
1893 template< typename MT // Type of the dense matrix
1894  , size_t... CSAs > // Compile time submatrix arguments
1896 {
1897  const size_t iend( row() + rows() );
1898  const size_t jend( column() + columns() );
1899  size_t nonzeros( 0UL );
1900 
1901  for( size_t i=row(); i<iend; ++i )
1902  for( size_t j=column(); j<jend; ++j )
1903  if( !isDefault( matrix_(i,j) ) )
1904  ++nonzeros;
1905 
1906  return nonzeros;
1907 }
1909 //*************************************************************************************************
1910 
1911 
1912 //*************************************************************************************************
1924 template< typename MT // Type of the dense matrix
1925  , size_t... CSAs > // Compile time submatrix arguments
1926 inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::nonZeros( size_t i ) const
1927 {
1928  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1929 
1930  const size_t jend( column() + columns() );
1931  size_t nonzeros( 0UL );
1932 
1933  for( size_t j=column(); j<jend; ++j )
1934  if( !isDefault( matrix_(row()+i,j) ) )
1935  ++nonzeros;
1936 
1937  return nonzeros;
1938 }
1940 //*************************************************************************************************
1941 
1942 
1943 //*************************************************************************************************
1949 template< typename MT // Type of the dense matrix
1950  , size_t... CSAs > // Compile time submatrix arguments
1952 {
1953  using blaze::clear;
1954 
1955  for( size_t i=row(); i<row()+rows(); ++i )
1956  {
1957  const size_t jbegin( ( IsUpper<MT>::value )
1958  ?( ( IsUniUpper<MT>::value || IsStrictlyUpper<MT>::value )
1959  ?( max( i+1UL, column() ) )
1960  :( max( i, column() ) ) )
1961  :( column() ) );
1962  const size_t jend ( ( IsLower<MT>::value )
1963  ?( ( IsUniLower<MT>::value || IsStrictlyLower<MT>::value )
1964  ?( min( i, column()+columns() ) )
1965  :( min( i+1UL, column()+columns() ) ) )
1966  :( column()+columns() ) );
1967 
1968  for( size_t j=jbegin; j<jend; ++j )
1969  clear( matrix_(i,j) );
1970  }
1971 }
1973 //*************************************************************************************************
1974 
1975 
1976 //*************************************************************************************************
1988 template< typename MT // Type of the dense matrix
1989  , size_t... CSAs > // Compile time submatrix arguments
1991 {
1992  using blaze::clear;
1993 
1994  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1995 
1996  const size_t jbegin( ( IsUpper<MT>::value )
1997  ?( ( IsUniUpper<MT>::value || IsStrictlyUpper<MT>::value )
1998  ?( max( i+1UL, column() ) )
1999  :( max( i, column() ) ) )
2000  :( column() ) );
2001  const size_t jend ( ( IsLower<MT>::value )
2002  ?( ( IsUniLower<MT>::value || IsStrictlyLower<MT>::value )
2003  ?( min( i, column()+columns() ) )
2004  :( min( i+1UL, column()+columns() ) ) )
2005  :( column()+columns() ) );
2006 
2007  for( size_t j=jbegin; j<jend; ++j )
2008  clear( matrix_(row()+i,j) );
2009 }
2011 //*************************************************************************************************
2012 
2013 
2014 //*************************************************************************************************
2024 template< typename MT // Type of the dense matrix
2025  , size_t... CSAs > // Compile time submatrix arguments
2026 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::hasOverlap() const noexcept
2027 {
2028  BLAZE_INTERNAL_ASSERT( IsSymmetric<MT>::value || IsHermitian<MT>::value, "Invalid matrix detected" );
2029 
2030  if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
2031  return false;
2032  else return true;
2033 }
2035 //*************************************************************************************************
2036 
2037 
2038 
2039 
2040 //=================================================================================================
2041 //
2042 // NUMERIC FUNCTIONS
2043 //
2044 //=================================================================================================
2045 
2046 //*************************************************************************************************
2064 template< typename MT // Type of the dense matrix
2065  , size_t... CSAs > // Compile time submatrix arguments
2066 inline Submatrix<MT,unaligned,false,true,CSAs...>&
2068 {
2069  if( rows() != columns() ) {
2070  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
2071  }
2072 
2073  if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
2074  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
2075  }
2076 
2077  decltype(auto) left( derestrict( *this ) );
2078  const ResultType tmp( trans( *this ) );
2079 
2080  smpAssign( left, tmp );
2081 
2082  return *this;
2083 }
2085 //*************************************************************************************************
2086 
2087 
2088 //*************************************************************************************************
2106 template< typename MT // Type of the dense matrix
2107  , size_t... CSAs > // Compile time submatrix arguments
2108 inline Submatrix<MT,unaligned,false,true,CSAs...>&
2109  Submatrix<MT,unaligned,false,true,CSAs...>::ctranspose()
2110 {
2111  if( rows() != columns() ) {
2112  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
2113  }
2114 
2115  if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
2116  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
2117  }
2118 
2119  decltype(auto) left( derestrict( *this ) );
2120  const ResultType tmp( ctrans( *this ) );
2121 
2122  smpAssign( left, tmp );
2123 
2124  return *this;
2125 }
2127 //*************************************************************************************************
2128 
2129 
2130 //*************************************************************************************************
2143 template< typename MT // Type of the dense matrix
2144  , size_t... CSAs > // Compile time submatrix arguments
2145 template< typename Other > // Data type of the scalar value
2146 inline Submatrix<MT,unaligned,false,true,CSAs...>&
2147  Submatrix<MT,unaligned,false,true,CSAs...>::scale( const Other& scalar )
2148 {
2150 
2151  const size_t iend( row() + rows() );
2152 
2153  for( size_t i=row(); i<iend; ++i )
2154  {
2155  const size_t jbegin( ( IsUpper<MT>::value )
2156  ?( ( IsStrictlyUpper<MT>::value )
2157  ?( max( i+1UL, column() ) )
2158  :( max( i, column() ) ) )
2159  :( column() ) );
2160  const size_t jend ( ( IsLower<MT>::value )
2161  ?( ( IsStrictlyLower<MT>::value )
2162  ?( min( i, column()+columns() ) )
2163  :( min( i+1UL, column()+columns() ) ) )
2164  :( column()+columns() ) );
2165 
2166  for( size_t j=jbegin; j<jend; ++j )
2167  matrix_(i,j) *= scalar;
2168  }
2169 
2170  return *this;
2171 }
2173 //*************************************************************************************************
2174 
2175 
2176 
2177 
2178 //=================================================================================================
2179 //
2180 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
2181 //
2182 //=================================================================================================
2183 
2184 //*************************************************************************************************
2195 template< typename MT // Type of the dense matrix
2196  , size_t... CSAs > // Compile time submatrix arguments
2197 template< typename Other > // Data type of the foreign expression
2198 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::canAlias( const Other* alias ) const noexcept
2199 {
2200  return matrix_.isAliased( alias );
2201 }
2203 //*************************************************************************************************
2204 
2205 
2206 //*************************************************************************************************
2217 template< typename MT // Type of the dense matrix
2218  , size_t... CSAs > // Compile time submatrix arguments
2219 template< typename MT2 // Data type of the foreign dense submatrix
2220  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
2221  , bool SO2 // Storage order of the foreign dense submatrix
2222  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
2223 inline bool
2224  Submatrix<MT,unaligned,false,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
2225 {
2226  return ( matrix_.isAliased( &alias->matrix_ ) &&
2227  ( row() + rows() > alias->row() ) &&
2228  ( row() < alias->row() + alias->rows() ) &&
2229  ( column() + columns() > alias->column() ) &&
2230  ( column() < alias->column() + alias->columns() ) );
2231 }
2233 //*************************************************************************************************
2234 
2235 
2236 //*************************************************************************************************
2247 template< typename MT // Type of the dense matrix
2248  , size_t... CSAs > // Compile time submatrix arguments
2249 template< typename Other > // Data type of the foreign expression
2250 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::isAliased( const Other* alias ) const noexcept
2251 {
2252  return matrix_.isAliased( alias );
2253 }
2255 //*************************************************************************************************
2256 
2257 
2258 //*************************************************************************************************
2269 template< typename MT // Type of the dense matrix
2270  , size_t... CSAs > // Compile time submatrix arguments
2271 template< typename MT2 // Data type of the foreign dense submatrix
2272  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
2273  , bool SO2 // Storage order of the foreign dense submatrix
2274  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
2275 inline bool
2276  Submatrix<MT,unaligned,false,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
2277 {
2278  return ( matrix_.isAliased( &alias->matrix_ ) &&
2279  ( row() + rows() > alias->row() ) &&
2280  ( row() < alias->row() + alias->rows() ) &&
2281  ( column() + columns() > alias->column() ) &&
2282  ( column() < alias->column() + alias->columns() ) );
2283 }
2285 //*************************************************************************************************
2286 
2287 
2288 //*************************************************************************************************
2298 template< typename MT // Type of the dense matrix
2299  , size_t... CSAs > // Compile time submatrix arguments
2300 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::isAligned() const noexcept
2301 {
2302  return isAligned_;
2303 }
2305 //*************************************************************************************************
2306 
2307 
2308 //*************************************************************************************************
2319 template< typename MT // Type of the dense matrix
2320  , size_t... CSAs > // Compile time submatrix arguments
2321 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::canSMPAssign() const noexcept
2322 {
2323  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
2324 }
2326 //*************************************************************************************************
2327 
2328 
2329 //*************************************************************************************************
2345 template< typename MT // Type of the dense matrix
2346  , size_t... CSAs > // Compile time submatrix arguments
2347 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,false,true,CSAs...>::SIMDType
2348  Submatrix<MT,unaligned,false,true,CSAs...>::load( size_t i, size_t j ) const noexcept
2349 {
2350  if( isAligned_ )
2351  return loada( i, j );
2352  else
2353  return loadu( i, j );
2354 }
2356 //*************************************************************************************************
2357 
2358 
2359 //*************************************************************************************************
2375 template< typename MT // Type of the dense matrix
2376  , size_t... CSAs > // Compile time submatrix arguments
2377 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,false,true,CSAs...>::SIMDType
2378  Submatrix<MT,unaligned,false,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
2379 {
2381 
2382  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2383  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2384  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2385  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2386 
2387  return matrix_.loada( row()+i, column()+j );
2388 }
2390 //*************************************************************************************************
2391 
2392 
2393 //*************************************************************************************************
2409 template< typename MT // Type of the dense matrix
2410  , size_t... CSAs > // Compile time submatrix arguments
2411 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,false,true,CSAs...>::SIMDType
2412  Submatrix<MT,unaligned,false,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
2413 {
2415 
2416  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2417  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2418  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2419  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2420 
2421  return matrix_.loadu( row()+i, column()+j );
2422 }
2424 //*************************************************************************************************
2425 
2426 
2427 //*************************************************************************************************
2444 template< typename MT // Type of the dense matrix
2445  , size_t... CSAs > // Compile time submatrix arguments
2447  Submatrix<MT,unaligned,false,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
2448 {
2449  if( isAligned_ )
2450  storea( i, j, value );
2451  else
2452  storeu( i, j, value );
2453 }
2455 //*************************************************************************************************
2456 
2457 
2458 //*************************************************************************************************
2475 template< typename MT // Type of the dense matrix
2476  , size_t... CSAs > // Compile time submatrix arguments
2478  Submatrix<MT,unaligned,false,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
2479 {
2481 
2482  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2483  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2484  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2485  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2486 
2487  matrix_.storea( row()+i, column()+j, value );
2488 }
2490 //*************************************************************************************************
2491 
2492 
2493 //*************************************************************************************************
2510 template< typename MT // Type of the dense matrix
2511  , size_t... CSAs > // Compile time submatrix arguments
2513  Submatrix<MT,unaligned,false,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
2514 {
2516 
2517  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2518  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2519  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2520  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2521 
2522  matrix_.storeu( row()+i, column()+j, value );
2523 }
2525 //*************************************************************************************************
2526 
2527 
2528 //*************************************************************************************************
2545 template< typename MT // Type of the dense matrix
2546  , size_t... CSAs > // Compile time submatrix arguments
2548  Submatrix<MT,unaligned,false,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
2549 {
2551 
2552  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2553  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2554  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2555  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2556 
2557  if( isAligned_ )
2558  matrix_.stream( row()+i, column()+j, value );
2559  else
2560  matrix_.storeu( row()+i, column()+j, value );
2561 }
2563 //*************************************************************************************************
2564 
2565 
2566 //*************************************************************************************************
2578 template< typename MT // Type of the dense matrix
2579  , size_t... CSAs > // Compile time submatrix arguments
2580 template< typename MT2 > // Type of the right-hand side dense matrix
2581 inline DisableIf_< typename Submatrix<MT,unaligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedAssign<MT2> >
2582  Submatrix<MT,unaligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
2583 {
2584  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2585  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2586 
2587  const size_t jpos( columns() & size_t(-2) );
2588  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
2589 
2590  for( size_t i=0UL; i<rows(); ++i ) {
2591  for( size_t j=0UL; j<jpos; j+=2UL ) {
2592  matrix_(row()+i,column()+j ) = (~rhs)(i,j );
2593  matrix_(row()+i,column()+j+1UL) = (~rhs)(i,j+1UL);
2594  }
2595  if( jpos < columns() ) {
2596  matrix_(row()+i,column()+jpos) = (~rhs)(i,jpos);
2597  }
2598  }
2599 }
2601 //*************************************************************************************************
2602 
2603 
2604 //*************************************************************************************************
2616 template< typename MT // Type of the dense matrix
2617  , size_t... CSAs > // Compile time submatrix arguments
2618 template< typename MT2 > // Type of the right-hand side dense matrix
2619 inline EnableIf_< typename Submatrix<MT,unaligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedAssign<MT2> >
2620  Submatrix<MT,unaligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
2621 {
2623 
2624  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2625  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2626 
2627  const size_t jpos( columns() & size_t(-SIMDSIZE) );
2628  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
2629 
2630  if( useStreaming && isAligned_ &&
2631  rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
2632  !(~rhs).isAliased( &matrix_ ) )
2633  {
2634  for( size_t i=0UL; i<rows(); ++i )
2635  {
2636  size_t j( 0UL );
2637  Iterator left( begin(i) );
2638  ConstIterator_<MT2> right( (~rhs).begin(i) );
2639 
2640  for( ; j<jpos; j+=SIMDSIZE ) {
2641  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2642  }
2643  for( ; j<columns(); ++j ) {
2644  *left = *right;
2645  }
2646  }
2647  }
2648  else
2649  {
2650  for( size_t i=0UL; i<rows(); ++i )
2651  {
2652  size_t j( 0UL );
2653  Iterator left( begin(i) );
2654  ConstIterator_<MT2> right( (~rhs).begin(i) );
2655 
2656  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
2657  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2658  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2659  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2660  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2661  }
2662  for( ; j<jpos; j+=SIMDSIZE ) {
2663  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2664  }
2665  for( ; j<columns(); ++j ) {
2666  *left = *right; ++left; ++right;
2667  }
2668  }
2669  }
2670 }
2672 //*************************************************************************************************
2673 
2674 
2675 //*************************************************************************************************
2687 template< typename MT // Type of the dense matrix
2688  , size_t... CSAs > // Compile time submatrix arguments
2689 template< typename MT2 > // Type of the right-hand side dense matrix
2690 inline void Submatrix<MT,unaligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
2691 {
2693 
2694  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2695  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2696 
2697  constexpr size_t block( BLOCK_SIZE );
2698 
2699  for( size_t ii=0UL; ii<rows(); ii+=block ) {
2700  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
2701  for( size_t jj=0UL; jj<columns(); jj+=block ) {
2702  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
2703  for( size_t i=ii; i<iend; ++i ) {
2704  for( size_t j=jj; j<jend; ++j ) {
2705  matrix_(row()+i,column()+j) = (~rhs)(i,j);
2706  }
2707  }
2708  }
2709  }
2710 }
2712 //*************************************************************************************************
2713 
2714 
2715 //*************************************************************************************************
2727 template< typename MT // Type of the dense matrix
2728  , size_t... CSAs > // Compile time submatrix arguments
2729 template< typename MT2 > // Type of the right-hand side sparse matrix
2730 inline void Submatrix<MT,unaligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
2731 {
2732  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2733  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2734 
2735  for( size_t i=0UL; i<rows(); ++i )
2736  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
2737  matrix_(row()+i,column()+element->index()) = element->value();
2738 }
2740 //*************************************************************************************************
2741 
2742 
2743 //*************************************************************************************************
2755 template< typename MT // Type of the dense matrix
2756  , size_t... CSAs > // Compile time submatrix arguments
2757 template< typename MT2 > // Type of the right-hand side sparse matrix
2758 inline void Submatrix<MT,unaligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
2759 {
2761 
2762  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2763  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2764 
2765  for( size_t j=0UL; j<columns(); ++j )
2766  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
2767  matrix_(row()+element->index(),column()+j) = element->value();
2768 }
2770 //*************************************************************************************************
2771 
2772 
2773 //*************************************************************************************************
2785 template< typename MT // Type of the dense matrix
2786  , size_t... CSAs > // Compile time submatrix arguments
2787 template< typename MT2 > // Type of the right-hand side dense matrix
2788 inline DisableIf_< typename Submatrix<MT,unaligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedAddAssign<MT2> >
2789  Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
2790 {
2791  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2792  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2793 
2794  const size_t jpos( columns() & size_t(-2) );
2795  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
2796 
2797  for( size_t i=0UL; i<rows(); ++i )
2798  {
2799  if( IsDiagonal<MT2>::value ) {
2800  matrix_(row()+i,column()+i) += (~rhs)(i,i);
2801  }
2802  else {
2803  for( size_t j=0UL; j<jpos; j+=2UL ) {
2804  matrix_(row()+i,column()+j ) += (~rhs)(i,j );
2805  matrix_(row()+i,column()+j+1UL) += (~rhs)(i,j+1UL);
2806  }
2807  if( jpos < columns() ) {
2808  matrix_(row()+i,column()+jpos) += (~rhs)(i,jpos);
2809  }
2810  }
2811  }
2812 }
2814 //*************************************************************************************************
2815 
2816 
2817 //*************************************************************************************************
2829 template< typename MT // Type of the dense matrix
2830  , size_t... CSAs > // Compile time submatrix arguments
2831 template< typename MT2 > // Type of the right-hand side dense matrix
2832 inline EnableIf_< typename Submatrix<MT,unaligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedAddAssign<MT2> >
2833  Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
2834 {
2836 
2837  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2838  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2839 
2840  for( size_t i=0UL; i<rows(); ++i )
2841  {
2842  const size_t jbegin( ( IsUpper<MT2>::value )
2843  ?( ( IsStrictlyUpper<MT2>::value ? i+1UL : i ) & size_t(-SIMDSIZE) )
2844  :( 0UL ) );
2845  const size_t jend ( ( IsLower<MT2>::value )
2846  ?( IsStrictlyLower<MT2>::value ? i : i+1UL )
2847  :( columns() ) );
2848  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
2849 
2850  const size_t jpos( jend & size_t(-SIMDSIZE) );
2851  BLAZE_INTERNAL_ASSERT( ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
2852 
2853  size_t j( jbegin );
2854  Iterator left( begin(i) + jbegin );
2855  ConstIterator_<MT2> right( (~rhs).begin(i) + jbegin );
2856 
2857  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
2858  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2859  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2860  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2861  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2862  }
2863  for( ; j<jpos; j+=SIMDSIZE ) {
2864  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2865  }
2866  for( ; j<jend; ++j ) {
2867  *left += *right; ++left; ++right;
2868  }
2869  }
2870 }
2872 //*************************************************************************************************
2873 
2874 
2875 //*************************************************************************************************
2887 template< typename MT // Type of the dense matrix
2888  , size_t... CSAs > // Compile time submatrix arguments
2889 template< typename MT2 > // Type of the right-hand side dense matrix
2890 inline void Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
2891 {
2893 
2894  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2895  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2896 
2897  constexpr size_t block( BLOCK_SIZE );
2898 
2899  for( size_t ii=0UL; ii<rows(); ii+=block ) {
2900  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
2901  for( size_t jj=0UL; jj<columns(); jj+=block ) {
2902  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
2903  for( size_t i=ii; i<iend; ++i ) {
2904  for( size_t j=jj; j<jend; ++j ) {
2905  matrix_(row()+i,column()+j) += (~rhs)(i,j);
2906  }
2907  }
2908  }
2909  }
2910 }
2912 //*************************************************************************************************
2913 
2914 
2915 //*************************************************************************************************
2927 template< typename MT // Type of the dense matrix
2928  , size_t... CSAs > // Compile time submatrix arguments
2929 template< typename MT2 > // Type of the right-hand side sparse matrix
2930 inline void Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
2931 {
2932  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2933  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2934 
2935  for( size_t i=0UL; i<rows(); ++i )
2936  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
2937  matrix_(row()+i,column()+element->index()) += element->value();
2938 }
2940 //*************************************************************************************************
2941 
2942 
2943 //*************************************************************************************************
2955 template< typename MT // Type of the dense matrix
2956  , size_t... CSAs > // Compile time submatrix arguments
2957 template< typename MT2 > // Type of the right-hand side sparse matrix
2958 inline void Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
2959 {
2961 
2962  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2963  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2964 
2965  for( size_t j=0UL; j<columns(); ++j )
2966  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
2967  matrix_(row()+element->index(),column()+j) += element->value();
2968 }
2970 //*************************************************************************************************
2971 
2972 
2973 //*************************************************************************************************
2985 template< typename MT // Type of the dense matrix
2986  , size_t... CSAs > // Compile time submatrix arguments
2987 template< typename MT2 > // Type of the right-hand side dense matrix
2988 inline DisableIf_< typename Submatrix<MT,unaligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedSubAssign<MT2> >
2989  Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
2990 {
2991  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2992  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2993 
2994  const size_t jpos( columns() & size_t(-2) );
2995  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
2996 
2997  for( size_t i=0UL; i<rows(); ++i )
2998  {
2999  if( IsDiagonal<MT2>::value ) {
3000  matrix_(row()+i,column()+i) -= (~rhs)(i,i);
3001  }
3002  else {
3003  for( size_t j=0UL; j<jpos; j+=2UL ) {
3004  matrix_(row()+i,column()+j ) -= (~rhs)(i,j );
3005  matrix_(row()+i,column()+j+1UL) -= (~rhs)(i,j+1UL);
3006  }
3007  if( jpos < columns() ) {
3008  matrix_(row()+i,column()+jpos) -= (~rhs)(i,jpos);
3009  }
3010  }
3011  }
3012 }
3014 //*************************************************************************************************
3015 
3016 
3017 //*************************************************************************************************
3029 template< typename MT // Type of the dense matrix
3030  , size_t... CSAs > // Compile time submatrix arguments
3031 template< typename MT2 > // Type of the right-hand side dense matrix
3032 inline EnableIf_< typename Submatrix<MT,unaligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedSubAssign<MT2> >
3033  Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
3034 {
3036 
3037  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3038  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3039 
3040  for( size_t i=0UL; i<rows(); ++i )
3041  {
3042  const size_t jbegin( ( IsUpper<MT2>::value )
3043  ?( ( IsStrictlyUpper<MT2>::value ? i+1UL : i ) & size_t(-SIMDSIZE) )
3044  :( 0UL ) );
3045  const size_t jend ( ( IsLower<MT2>::value )
3046  ?( IsStrictlyLower<MT2>::value ? i : i+1UL )
3047  :( columns() ) );
3048  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
3049 
3050  const size_t jpos( jend & size_t(-SIMDSIZE) );
3051  BLAZE_INTERNAL_ASSERT( ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
3052 
3053  size_t j( jbegin );
3054  Iterator left( begin(i) + jbegin );
3055  ConstIterator_<MT2> right( (~rhs).begin(i) + jbegin );
3056 
3057  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
3058  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3059  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3060  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3061  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3062  }
3063  for( ; j<jpos; j+=SIMDSIZE ) {
3064  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3065  }
3066  for( ; j<jend; ++j ) {
3067  *left -= *right; ++left; ++right;
3068  }
3069  }
3070 }
3072 //*************************************************************************************************
3073 
3074 
3075 //*************************************************************************************************
3087 template< typename MT // Type of the dense matrix
3088  , size_t... CSAs > // Compile time submatrix arguments
3089 template< typename MT2 > // Type of the right-hand side dense matrix
3090 inline void Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
3091 {
3093 
3094  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3095  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3096 
3097  constexpr size_t block( BLOCK_SIZE );
3098 
3099  for( size_t ii=0UL; ii<rows(); ii+=block ) {
3100  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
3101  for( size_t jj=0UL; jj<columns(); jj+=block ) {
3102  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
3103  for( size_t i=ii; i<iend; ++i ) {
3104  for( size_t j=jj; j<jend; ++j ) {
3105  matrix_(row()+i,column()+j) -= (~rhs)(i,j);
3106  }
3107  }
3108  }
3109  }
3110 }
3112 //*************************************************************************************************
3113 
3114 
3115 //*************************************************************************************************
3127 template< typename MT // Type of the dense matrix
3128  , size_t... CSAs > // Compile time submatrix arguments
3129 template< typename MT2 > // Type of the right-hand side sparse matrix
3130 inline void Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
3131 {
3132  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3133  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3134 
3135  for( size_t i=0UL; i<rows(); ++i )
3136  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
3137  matrix_(row()+i,column()+element->index()) -= element->value();
3138 }
3140 //*************************************************************************************************
3141 
3142 
3143 //*************************************************************************************************
3155 template< typename MT // Type of the dense matrix
3156  , size_t... CSAs > // Compile time submatrix arguments
3157 template< typename MT2 > // Type of the right-hand side sparse matrix
3158 inline void Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
3159 {
3161 
3162  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3163  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3164 
3165  for( size_t j=0UL; j<columns(); ++j )
3166  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
3167  matrix_(row()+element->index(),column()+j) -= element->value();
3168 }
3170 //*************************************************************************************************
3171 
3172 
3173 //*************************************************************************************************
3185 template< typename MT // Type of the dense matrix
3186  , size_t... CSAs > // Compile time submatrix arguments
3187 template< typename MT2 > // Type of the right-hand side dense matrix
3188 inline DisableIf_< typename Submatrix<MT,unaligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedSchurAssign<MT2> >
3189  Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
3190 {
3191  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3192  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3193 
3194  const size_t jpos( columns() & size_t(-2) );
3195  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
3196 
3197  for( size_t i=0UL; i<rows(); ++i ) {
3198  for( size_t j=0UL; j<jpos; j+=2UL ) {
3199  matrix_(row()+i,column()+j ) *= (~rhs)(i,j );
3200  matrix_(row()+i,column()+j+1UL) *= (~rhs)(i,j+1UL);
3201  }
3202  if( jpos < columns() ) {
3203  matrix_(row()+i,column()+jpos) *= (~rhs)(i,jpos);
3204  }
3205  }
3206 }
3208 //*************************************************************************************************
3209 
3210 
3211 //*************************************************************************************************
3223 template< typename MT // Type of the dense matrix
3224  , size_t... CSAs > // Compile time submatrix arguments
3225 template< typename MT2 > // Type of the right-hand side dense matrix
3226 inline EnableIf_< typename Submatrix<MT,unaligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedSchurAssign<MT2> >
3227  Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
3228 {
3230 
3231  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3232  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3233 
3234  for( size_t i=0UL; i<rows(); ++i )
3235  {
3236  const size_t jpos( columns() & size_t(-SIMDSIZE) );
3237  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
3238 
3239  size_t j( 0UL );
3240  Iterator left( begin(i) );
3241  ConstIterator_<MT2> right( (~rhs).begin(i) );
3242 
3243  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
3244  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3245  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3246  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3247  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3248  }
3249  for( ; j<jpos; j+=SIMDSIZE ) {
3250  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3251  }
3252  for( ; j<columns(); ++j ) {
3253  *left *= *right; ++left; ++right;
3254  }
3255  }
3256 }
3258 //*************************************************************************************************
3259 
3260 
3261 //*************************************************************************************************
3273 template< typename MT // Type of the dense matrix
3274  , size_t... CSAs > // Compile time submatrix arguments
3275 template< typename MT2 > // Type of the right-hand side dense matrix
3276 inline void Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
3277 {
3279 
3280  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3281  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3282 
3283  constexpr size_t block( BLOCK_SIZE );
3284 
3285  for( size_t ii=0UL; ii<rows(); ii+=block ) {
3286  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
3287  for( size_t jj=0UL; jj<columns(); jj+=block ) {
3288  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
3289  for( size_t i=ii; i<iend; ++i ) {
3290  for( size_t j=jj; j<jend; ++j ) {
3291  matrix_(row()+i,column()+j) *= (~rhs)(i,j);
3292  }
3293  }
3294  }
3295  }
3296 }
3298 //*************************************************************************************************
3299 
3300 
3301 //*************************************************************************************************
3313 template< typename MT // Type of the dense matrix
3314  , size_t... CSAs > // Compile time submatrix arguments
3315 template< typename MT2 > // Type of the right-hand side sparse matrix
3316 inline void Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
3317 {
3318  using blaze::reset;
3319 
3320  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3321  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3322 
3323  for( size_t i=0UL; i<rows(); ++i )
3324  {
3325  size_t j( 0UL );
3326 
3327  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element ) {
3328  for( ; j<element->index(); ++j )
3329  reset( matrix_(row()+i,column()+j) );
3330  matrix_(row()+i,column()+j) *= element->value();
3331  ++j;
3332  }
3333 
3334  for( ; j<columns(); ++j ) {
3335  reset( matrix_(row()+i,column()+j) );
3336  }
3337  }
3338 }
3340 //*************************************************************************************************
3341 
3342 
3343 //*************************************************************************************************
3355 template< typename MT // Type of the dense matrix
3356  , size_t... CSAs > // Compile time submatrix arguments
3357 template< typename MT2 > // Type of the right-hand side sparse matrix
3358 inline void Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
3359 {
3360  using blaze::reset;
3361 
3363 
3364  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3365  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3366 
3367  for( size_t j=0UL; j<columns(); ++j )
3368  {
3369  size_t i( 0UL );
3370 
3371  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element ) {
3372  for( ; i<element->index(); ++i )
3373  reset( matrix_(row()+i,column()+j) );
3374  matrix_(row()+i,column()+j) *= element->value();
3375  ++i;
3376  }
3377 
3378  for( ; i<rows(); ++i ) {
3379  reset( matrix_(row()+i,column()+j) );
3380  }
3381  }
3382 }
3384 //*************************************************************************************************
3385 
3386 
3387 
3388 
3389 
3390 
3391 
3392 
3393 //=================================================================================================
3394 //
3395 // CLASS TEMPLATE SPECIALIZATION FOR UNALIGNED COLUMN-MAJOR DENSE SUBMATRICES
3396 //
3397 //=================================================================================================
3398 
3399 //*************************************************************************************************
3407 template< typename MT // Type of the dense matrix
3408  , size_t... CSAs > // Compile time submatrix arguments
3409 class Submatrix<MT,unaligned,true,true,CSAs...>
3410  : public View< DenseMatrix< Submatrix<MT,unaligned,true,true,CSAs...>, true > >
3411  , private SubmatrixData<CSAs...>
3412 {
3413  private:
3414  //**Type definitions****************************************************************************
3415  using DataType = SubmatrixData<CSAs...>;
3416  using Operand = If_< IsExpression<MT>, MT, MT& >;
3417  //**********************************************************************************************
3418 
3419  public:
3420  //**Type definitions****************************************************************************
3422  using This = Submatrix<MT,unaligned,true,true,CSAs...>;
3423 
3424  using BaseType = DenseMatrix<This,true>;
3425  using ViewedType = MT;
3426  using ResultType = SubmatrixTrait_<MT,CSAs...>;
3427  using OppositeType = OppositeType_<ResultType>;
3428  using TransposeType = TransposeType_<ResultType>;
3429  using ElementType = ElementType_<MT>;
3430  using SIMDType = SIMDTrait_<ElementType>;
3431  using ReturnType = ReturnType_<MT>;
3432  using CompositeType = const Submatrix&;
3433 
3435  using ConstReference = ConstReference_<MT>;
3436 
3438  using Reference = If_< IsConst<MT>, ConstReference, Reference_<MT> >;
3439 
3441  using ConstPointer = ConstPointer_<MT>;
3442 
3444  using Pointer = If_< Or< IsConst<MT>, Not< HasMutableDataAccess<MT> > >, ConstPointer, Pointer_<MT> >;
3445  //**********************************************************************************************
3446 
3447  //**SubmatrixIterator class definition**********************************************************
3450  template< typename IteratorType > // Type of the dense matrix iterator
3451  class SubmatrixIterator
3452  {
3453  public:
3454  //**Type definitions*************************************************************************
3456  using IteratorCategory = typename std::iterator_traits<IteratorType>::iterator_category;
3457 
3459  using ValueType = typename std::iterator_traits<IteratorType>::value_type;
3460 
3462  using PointerType = typename std::iterator_traits<IteratorType>::pointer;
3463 
3465  using ReferenceType = typename std::iterator_traits<IteratorType>::reference;
3466 
3468  using DifferenceType = typename std::iterator_traits<IteratorType>::difference_type;
3469 
3470  // STL iterator requirements
3471  using iterator_category = IteratorCategory;
3472  using value_type = ValueType;
3473  using pointer = PointerType;
3474  using reference = ReferenceType;
3475  using difference_type = DifferenceType;
3476  //*******************************************************************************************
3477 
3478  //**Constructor******************************************************************************
3481  inline SubmatrixIterator()
3482  : iterator_ ( ) // Iterator to the current submatrix element
3483  , isAligned_( false ) // Memory alignment flag
3484  {}
3485  //*******************************************************************************************
3486 
3487  //**Constructor******************************************************************************
3495  inline SubmatrixIterator( IteratorType iterator, bool isMemoryAligned )
3496  : iterator_ ( iterator ) // Iterator to the current submatrix element
3497  , isAligned_( isMemoryAligned ) // Memory alignment flag
3498  {}
3499  //*******************************************************************************************
3500 
3501  //**Constructor******************************************************************************
3506  template< typename IteratorType2 >
3507  inline SubmatrixIterator( const SubmatrixIterator<IteratorType2>& it )
3508  : iterator_ ( it.base() ) // Iterator to the current submatrix element
3509  , isAligned_( it.isAligned() ) // Memory alignment flag
3510  {}
3511  //*******************************************************************************************
3512 
3513  //**Addition assignment operator*************************************************************
3519  inline SubmatrixIterator& operator+=( size_t inc ) {
3520  iterator_ += inc;
3521  return *this;
3522  }
3523  //*******************************************************************************************
3524 
3525  //**Subtraction assignment operator**********************************************************
3531  inline SubmatrixIterator& operator-=( size_t dec ) {
3532  iterator_ -= dec;
3533  return *this;
3534  }
3535  //*******************************************************************************************
3536 
3537  //**Prefix increment operator****************************************************************
3542  inline SubmatrixIterator& operator++() {
3543  ++iterator_;
3544  return *this;
3545  }
3546  //*******************************************************************************************
3547 
3548  //**Postfix increment operator***************************************************************
3553  inline const SubmatrixIterator operator++( int ) {
3554  return SubmatrixIterator( iterator_++, isAligned_ );
3555  }
3556  //*******************************************************************************************
3557 
3558  //**Prefix decrement operator****************************************************************
3563  inline SubmatrixIterator& operator--() {
3564  --iterator_;
3565  return *this;
3566  }
3567  //*******************************************************************************************
3568 
3569  //**Postfix decrement operator***************************************************************
3574  inline const SubmatrixIterator operator--( int ) {
3575  return SubmatrixIterator( iterator_--, isAligned_ );
3576  }
3577  //*******************************************************************************************
3578 
3579  //**Element access operator******************************************************************
3584  inline ReferenceType operator*() const {
3585  return *iterator_;
3586  }
3587  //*******************************************************************************************
3588 
3589  //**Element access operator******************************************************************
3594  inline IteratorType operator->() const {
3595  return iterator_;
3596  }
3597  //*******************************************************************************************
3598 
3599  //**Load function****************************************************************************
3609  inline SIMDType load() const noexcept {
3610  if( isAligned_ )
3611  return loada();
3612  else
3613  return loadu();
3614  }
3615  //*******************************************************************************************
3616 
3617  //**Loada function***************************************************************************
3627  inline SIMDType loada() const noexcept {
3628  return iterator_.loada();
3629  }
3630  //*******************************************************************************************
3631 
3632  //**Loadu function***************************************************************************
3642  inline SIMDType loadu() const noexcept {
3643  return iterator_.loadu();
3644  }
3645  //*******************************************************************************************
3646 
3647  //**Store function***************************************************************************
3658  inline void store( const SIMDType& value ) const {
3659  storeu( value );
3660  }
3661  //*******************************************************************************************
3662 
3663  //**Storea function**************************************************************************
3674  inline void storea( const SIMDType& value ) const {
3675  iterator_.storea( value );
3676  }
3677  //*******************************************************************************************
3678 
3679  //**Storeu function**************************************************************************
3690  inline void storeu( const SIMDType& value ) const {
3691  if( isAligned_ ) {
3692  iterator_.storea( value );
3693  }
3694  else {
3695  iterator_.storeu( value );
3696  }
3697  }
3698  //*******************************************************************************************
3699 
3700  //**Stream function**************************************************************************
3711  inline void stream( const SIMDType& value ) const {
3712  iterator_.stream( value );
3713  }
3714  //*******************************************************************************************
3715 
3716  //**Equality operator************************************************************************
3722  inline bool operator==( const SubmatrixIterator& rhs ) const {
3723  return iterator_ == rhs.iterator_;
3724  }
3725  //*******************************************************************************************
3726 
3727  //**Inequality operator**********************************************************************
3733  inline bool operator!=( const SubmatrixIterator& rhs ) const {
3734  return iterator_ != rhs.iterator_;
3735  }
3736  //*******************************************************************************************
3737 
3738  //**Less-than operator***********************************************************************
3744  inline bool operator<( const SubmatrixIterator& rhs ) const {
3745  return iterator_ < rhs.iterator_;
3746  }
3747  //*******************************************************************************************
3748 
3749  //**Greater-than operator********************************************************************
3755  inline bool operator>( const SubmatrixIterator& rhs ) const {
3756  return iterator_ > rhs.iterator_;
3757  }
3758  //*******************************************************************************************
3759 
3760  //**Less-or-equal-than operator**************************************************************
3766  inline bool operator<=( const SubmatrixIterator& rhs ) const {
3767  return iterator_ <= rhs.iterator_;
3768  }
3769  //*******************************************************************************************
3770 
3771  //**Greater-or-equal-than operator***********************************************************
3777  inline bool operator>=( const SubmatrixIterator& rhs ) const {
3778  return iterator_ >= rhs.iterator_;
3779  }
3780  //*******************************************************************************************
3781 
3782  //**Subtraction operator*********************************************************************
3788  inline DifferenceType operator-( const SubmatrixIterator& rhs ) const {
3789  return iterator_ - rhs.iterator_;
3790  }
3791  //*******************************************************************************************
3792 
3793  //**Addition operator************************************************************************
3800  friend inline const SubmatrixIterator operator+( const SubmatrixIterator& it, size_t inc ) {
3801  return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
3802  }
3803  //*******************************************************************************************
3804 
3805  //**Addition operator************************************************************************
3812  friend inline const SubmatrixIterator operator+( size_t inc, const SubmatrixIterator& it ) {
3813  return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
3814  }
3815  //*******************************************************************************************
3816 
3817  //**Subtraction operator*********************************************************************
3824  friend inline const SubmatrixIterator operator-( const SubmatrixIterator& it, size_t dec ) {
3825  return SubmatrixIterator( it.iterator_ - dec, it.isAligned_ );
3826  }
3827  //*******************************************************************************************
3828 
3829  //**Base function****************************************************************************
3834  inline IteratorType base() const {
3835  return iterator_;
3836  }
3837  //*******************************************************************************************
3838 
3839  //**IsAligned function***********************************************************************
3844  inline bool isAligned() const noexcept {
3845  return isAligned_;
3846  }
3847  //*******************************************************************************************
3848 
3849  private:
3850  //**Member variables*************************************************************************
3851  IteratorType iterator_;
3852  bool isAligned_;
3853  //*******************************************************************************************
3854  };
3855  //**********************************************************************************************
3856 
3857  //**Type definitions****************************************************************************
3859  using ConstIterator = SubmatrixIterator< ConstIterator_<MT> >;
3860 
3862  using Iterator = If_< IsConst<MT>, ConstIterator, SubmatrixIterator< Iterator_<MT> > >;
3863  //**********************************************************************************************
3864 
3865  //**Compilation flags***************************************************************************
3867  enum : bool { simdEnabled = MT::simdEnabled };
3868 
3870  enum : bool { smpAssignable = MT::smpAssignable };
3871  //**********************************************************************************************
3872 
3873  //**Constructors********************************************************************************
3876  template< typename... RSAs >
3877  explicit inline Submatrix( MT& matrix, RSAs... args );
3878  // No explicitly declared copy constructor.
3880  //**********************************************************************************************
3881 
3882  //**Destructor**********************************************************************************
3883  // No explicitly declared destructor.
3884  //**********************************************************************************************
3885 
3886  //**Data access functions***********************************************************************
3889  inline Reference operator()( size_t i, size_t j );
3890  inline ConstReference operator()( size_t i, size_t j ) const;
3891  inline Reference at( size_t i, size_t j );
3892  inline ConstReference at( size_t i, size_t j ) const;
3893  inline Pointer data () noexcept;
3894  inline ConstPointer data () const noexcept;
3895  inline Pointer data ( size_t j ) noexcept;
3896  inline ConstPointer data ( size_t j ) const noexcept;
3897  inline Iterator begin ( size_t j );
3898  inline ConstIterator begin ( size_t j ) const;
3899  inline ConstIterator cbegin( size_t j ) const;
3900  inline Iterator end ( size_t j );
3901  inline ConstIterator end ( size_t j ) const;
3902  inline ConstIterator cend ( size_t j ) const;
3904  //**********************************************************************************************
3905 
3906  //**Assignment operators************************************************************************
3909  inline Submatrix& operator=( const ElementType& rhs );
3910  inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
3911  inline Submatrix& operator=( const Submatrix& rhs );
3912 
3913  template< typename MT2, bool SO >
3914  inline Submatrix& operator=( const Matrix<MT2,SO>& rhs );
3915 
3916  template< typename MT2, bool SO >
3917  inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
3918  operator+=( const Matrix<MT2,SO>& rhs );
3919 
3920  template< typename MT2, bool SO >
3921  inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
3922  operator+=( const Matrix<MT2,SO>& rhs );
3923 
3924  template< typename MT2, bool SO >
3925  inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
3926  operator-=( const Matrix<MT2,SO>& rhs );
3927 
3928  template< typename MT2, bool SO >
3929  inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
3930  operator-=( const Matrix<MT2,SO>& rhs );
3931 
3932  template< typename MT2, bool SO >
3933  inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
3934  operator%=( const Matrix<MT2,SO>& rhs );
3935 
3936  template< typename MT2, bool SO >
3937  inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
3938  operator%=( const Matrix<MT2,SO>& rhs );
3940  //**********************************************************************************************
3941 
3942  //**Utility functions***************************************************************************
3945  using DataType::row;
3946  using DataType::column;
3947  using DataType::rows;
3948  using DataType::columns;
3949 
3950  inline MT& operand() noexcept;
3951  inline const MT& operand() const noexcept;
3952 
3953  inline size_t spacing() const noexcept;
3954  inline size_t capacity() const noexcept;
3955  inline size_t capacity( size_t i ) const noexcept;
3956  inline size_t nonZeros() const;
3957  inline size_t nonZeros( size_t i ) const;
3958  inline void reset();
3959  inline void reset( size_t i );
3961  //**********************************************************************************************
3962 
3963  //**Numeric functions***************************************************************************
3966  inline Submatrix& transpose();
3967  inline Submatrix& ctranspose();
3968 
3969  template< typename Other > inline Submatrix& scale( const Other& scalar );
3971  //**********************************************************************************************
3972 
3973  private:
3974  //**********************************************************************************************
3976  template< typename MT2 >
3977  struct VectorizedAssign {
3978  enum : bool { value = useOptimizedKernels &&
3979  simdEnabled && MT2::simdEnabled &&
3980  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value };
3981  };
3982  //**********************************************************************************************
3983 
3984  //**********************************************************************************************
3986  template< typename MT2 >
3987  struct VectorizedAddAssign {
3988  enum : bool { value = useOptimizedKernels &&
3989  simdEnabled && MT2::simdEnabled &&
3990  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value &&
3991  HasSIMDAdd< ElementType, ElementType_<MT2> >::value &&
3992  !IsDiagonal<MT2>::value };
3993  };
3994  //**********************************************************************************************
3995 
3996  //**********************************************************************************************
3998  template< typename MT2 >
3999  struct VectorizedSubAssign {
4000  enum : bool { value = useOptimizedKernels &&
4001  simdEnabled && MT2::simdEnabled &&
4002  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value &&
4003  HasSIMDSub< ElementType, ElementType_<MT2> >::value &&
4004  !IsDiagonal<MT2>::value };
4005  };
4006  //**********************************************************************************************
4007 
4008  //**********************************************************************************************
4010  template< typename MT2 >
4011  struct VectorizedSchurAssign {
4012  enum : bool { value = useOptimizedKernels &&
4013  simdEnabled && MT2::simdEnabled &&
4014  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value &&
4015  HasSIMDMult< ElementType, ElementType_<MT2> >::value };
4016  };
4017  //**********************************************************************************************
4018 
4019  //**SIMD properties*****************************************************************************
4021  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
4022  //**********************************************************************************************
4023 
4024  public:
4025  //**Expression template evaluation functions****************************************************
4028  template< typename Other >
4029  inline bool canAlias( const Other* alias ) const noexcept;
4030 
4031  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
4032  inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
4033 
4034  template< typename Other >
4035  inline bool isAliased( const Other* alias ) const noexcept;
4036 
4037  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
4038  inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
4039 
4040  inline bool isAligned () const noexcept;
4041  inline bool canSMPAssign() const noexcept;
4042 
4043  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
4044  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
4045  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
4046 
4047  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
4048  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
4049  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
4050  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
4051 
4052  template< typename MT2 >
4053  inline DisableIf_< VectorizedAssign<MT2> > assign( const DenseMatrix<MT2,true>& rhs );
4054 
4055  template< typename MT2 >
4056  inline EnableIf_< VectorizedAssign<MT2> > assign( const DenseMatrix<MT2,true>& rhs );
4057 
4058  template< typename MT2 > inline void assign( const DenseMatrix<MT2,false>& rhs );
4059  template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
4060  template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
4061 
4062  template< typename MT2 >
4063  inline DisableIf_< VectorizedAddAssign<MT2> > addAssign( const DenseMatrix<MT2,true>& rhs );
4064 
4065  template< typename MT2 >
4066  inline EnableIf_< VectorizedAddAssign<MT2> > addAssign( const DenseMatrix<MT2,true>& rhs );
4067 
4068  template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,false>& rhs );
4069  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
4070  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
4071 
4072  template< typename MT2 >
4073  inline DisableIf_< VectorizedSubAssign<MT2> > subAssign( const DenseMatrix<MT2,true>& rhs );
4074 
4075  template< typename MT2 >
4076  inline EnableIf_< VectorizedSubAssign<MT2> > subAssign( const DenseMatrix<MT2,true>& rhs );
4077 
4078  template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,false>& rhs );
4079  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
4080  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
4081 
4082  template< typename MT2 >
4083  inline DisableIf_< VectorizedSchurAssign<MT2> > schurAssign( const DenseMatrix<MT2,true>& rhs );
4084 
4085  template< typename MT2 >
4086  inline EnableIf_< VectorizedSchurAssign<MT2> > schurAssign( const DenseMatrix<MT2,true>& rhs );
4087 
4088  template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,false>& rhs );
4089  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
4090  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
4092  //**********************************************************************************************
4093 
4094  private:
4095  //**Utility functions***************************************************************************
4098  inline bool hasOverlap() const noexcept;
4100  //**********************************************************************************************
4101 
4102  //**Member variables****************************************************************************
4105  Operand matrix_;
4106  const bool isAligned_;
4107 
4114  //**********************************************************************************************
4115 
4116  //**Friend declarations*************************************************************************
4117  template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
4118  //**********************************************************************************************
4119 
4120  //**Compile time checks*************************************************************************
4128  //**********************************************************************************************
4129 };
4131 //*************************************************************************************************
4132 
4133 
4134 
4135 
4136 //=================================================================================================
4137 //
4138 // CONSTRUCTORS
4139 //
4140 //=================================================================================================
4141 
4142 //*************************************************************************************************
4155 template< typename MT // Type of the dense matrix
4156  , size_t... CSAs > // Compile time submatrix arguments
4157 template< typename... RSAs > // Runtime submatrix arguments
4158 inline Submatrix<MT,unaligned,true,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
4159  : DataType ( args... ) // Base class initialization
4160  , matrix_ ( matrix ) // The matrix containing the submatrix
4161  , isAligned_( simdEnabled && matrix.data() != nullptr && checkAlignment( data() ) &&
4162  ( columns() < 2UL || ( matrix.spacing() & size_t(-SIMDSIZE) ) == 0UL ) )
4163 {
4164  if( !Contains< TypeList<RSAs...>, Unchecked >::value ) {
4165  if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
4166  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
4167  }
4168  }
4169  else {
4170  BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
4171  BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
4172  }
4173 }
4175 //*************************************************************************************************
4176 
4177 
4178 
4179 
4180 //=================================================================================================
4181 //
4182 // DATA ACCESS FUNCTIONS
4183 //
4184 //=================================================================================================
4185 
4186 //*************************************************************************************************
4197 template< typename MT // Type of the dense matrix
4198  , size_t... CSAs > // Compile time submatrix arguments
4199 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Reference
4200  Submatrix<MT,unaligned,true,true,CSAs...>::operator()( size_t i, size_t j )
4201 {
4202  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
4203  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
4204 
4205  return matrix_(row()+i,column()+j);
4206 }
4208 //*************************************************************************************************
4209 
4210 
4211 //*************************************************************************************************
4222 template< typename MT // Type of the dense matrix
4223  , size_t... CSAs > // Compile time submatrix arguments
4224 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstReference
4225  Submatrix<MT,unaligned,true,true,CSAs...>::operator()( size_t i, size_t j ) const
4226 {
4227  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
4228  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
4229 
4230  return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
4231 }
4233 //*************************************************************************************************
4234 
4235 
4236 //*************************************************************************************************
4248 template< typename MT // Type of the dense matrix
4249  , size_t... CSAs > // Compile time submatrix arguments
4250 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Reference
4251  Submatrix<MT,unaligned,true,true,CSAs...>::at( size_t i, size_t j )
4252 {
4253  if( i >= rows() ) {
4254  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
4255  }
4256  if( j >= columns() ) {
4257  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
4258  }
4259  return (*this)(i,j);
4260 }
4262 //*************************************************************************************************
4263 
4264 
4265 //*************************************************************************************************
4277 template< typename MT // Type of the dense matrix
4278  , size_t... CSAs > // Compile time submatrix arguments
4279 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstReference
4280  Submatrix<MT,unaligned,true,true,CSAs...>::at( size_t i, size_t j ) const
4281 {
4282  if( i >= rows() ) {
4283  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
4284  }
4285  if( j >= columns() ) {
4286  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
4287  }
4288  return (*this)(i,j);
4289 }
4291 //*************************************************************************************************
4292 
4293 
4294 //*************************************************************************************************
4304 template< typename MT // Type of the dense matrix
4305  , size_t... CSAs > // Compile time submatrix arguments
4306 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Pointer
4308 {
4309  return matrix_.data() + row() + column()*spacing();
4310 }
4312 //*************************************************************************************************
4313 
4314 
4315 //*************************************************************************************************
4325 template< typename MT // Type of the dense matrix
4326  , size_t... CSAs > // Compile time submatrix arguments
4327 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstPointer
4329 {
4330  return matrix_.data() + row() + column()*spacing();
4331 }
4333 //*************************************************************************************************
4334 
4335 
4336 //*************************************************************************************************
4345 template< typename MT // Type of the dense matrix
4346  , size_t... CSAs > // Compile time submatrix arguments
4347 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Pointer
4349 {
4350  return matrix_.data() + row() + (column()+j)*spacing();
4351 }
4353 //*************************************************************************************************
4354 
4355 
4356 //*************************************************************************************************
4365 template< typename MT // Type of the dense matrix
4366  , size_t... CSAs > // Compile time submatrix arguments
4367 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstPointer
4368  Submatrix<MT,unaligned,true,true,CSAs...>::data( size_t j ) const noexcept
4369 {
4370  return matrix_.data() + row() + (column()+j)*spacing();
4371 }
4373 //*************************************************************************************************
4374 
4375 
4376 //*************************************************************************************************
4383 template< typename MT // Type of the dense matrix
4384  , size_t... CSAs > // Compile time submatrix arguments
4385 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Iterator
4387 {
4388  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4389  return Iterator( matrix_.begin( column() + j ) + row(), isAligned_ );
4390 }
4392 //*************************************************************************************************
4393 
4394 
4395 //*************************************************************************************************
4402 template< typename MT // Type of the dense matrix
4403  , size_t... CSAs > // Compile time submatrix arguments
4404 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4406 {
4407  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4408  return ConstIterator( matrix_.cbegin( column() + j ) + row(), isAligned_ );
4409 }
4411 //*************************************************************************************************
4412 
4413 
4414 //*************************************************************************************************
4421 template< typename MT // Type of the dense matrix
4422  , size_t... CSAs > // Compile time submatrix arguments
4423 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4425 {
4426  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4427  return ConstIterator( matrix_.cbegin( column() + j ) + row(), isAligned_ );
4428 }
4430 //*************************************************************************************************
4431 
4432 
4433 //*************************************************************************************************
4440 template< typename MT // Type of the dense matrix
4441  , size_t... CSAs > // Compile time submatrix arguments
4442 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Iterator
4444 {
4445  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4446  return Iterator( matrix_.begin( column() + j ) + row() + rows(), isAligned_ );
4447 }
4449 //*************************************************************************************************
4450 
4451 
4452 //*************************************************************************************************
4459 template< typename MT // Type of the dense matrix
4460  , size_t... CSAs > // Compile time submatrix arguments
4461 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4463 {
4464  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4465  return ConstIterator( matrix_.cbegin( column() + j ) + row() + rows(), isAligned_ );
4466 }
4468 //*************************************************************************************************
4469 
4470 
4471 //*************************************************************************************************
4478 template< typename MT // Type of the dense matrix
4479  , size_t... CSAs > // Compile time submatrix arguments
4480 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4482 {
4483  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4484  return ConstIterator( matrix_.cbegin( column() + j ) + row() + rows(), isAligned_ );
4485 }
4487 //*************************************************************************************************
4488 
4489 
4490 
4491 
4492 //=================================================================================================
4493 //
4494 // ASSIGNMENT OPERATORS
4495 //
4496 //=================================================================================================
4497 
4498 //*************************************************************************************************
4509 template< typename MT // Type of the dense matrix
4510  , size_t... CSAs > // Compile time submatrix arguments
4511 inline Submatrix<MT,unaligned,true,true,CSAs...>&
4512  Submatrix<MT,unaligned,true,true,CSAs...>::operator=( const ElementType& rhs )
4513 {
4514  const size_t jend( column() + columns() );
4515  decltype(auto) left( derestrict( matrix_ ) );
4516 
4517  for( size_t j=column(); j<jend; ++j )
4518  {
4519  const size_t ibegin( ( IsLower<MT>::value )
4520  ?( ( IsUniLower<MT>::value || IsStrictlyLower<MT>::value )
4521  ?( max( j+1UL, row() ) )
4522  :( max( j, row() ) ) )
4523  :( row() ) );
4524  const size_t iend ( ( IsUpper<MT>::value )
4525  ?( ( IsUniUpper<MT>::value || IsStrictlyUpper<MT>::value )
4526  ?( min( j, row()+rows() ) )
4527  :( min( j+1UL, row()+rows() ) ) )
4528  :( row()+rows() ) );
4529 
4530  for( size_t i=ibegin; i<iend; ++i ) {
4531  if( !IsRestricted<MT>::value || IsTriangular<MT>::value || trySet( matrix_, i, j, rhs ) )
4532  left(i,j) = rhs;
4533  }
4534  }
4535 
4536  return *this;
4537 }
4539 //*************************************************************************************************
4540 
4541 
4542 //*************************************************************************************************
4558 template< typename MT // Type of the dense matrix
4559  , size_t... CSAs > // Compile time submatrix arguments
4560 inline Submatrix<MT,unaligned,true,true,CSAs...>&
4561  Submatrix<MT,unaligned,true,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
4562 {
4563  using blaze::reset;
4564 
4565  if( list.size() != rows() ) {
4566  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
4567  }
4568 
4569  if( IsRestricted<MT>::value ) {
4570  const InitializerMatrix<ElementType> tmp( list, columns() );
4571  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
4572  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4573  }
4574  }
4575 
4576  decltype(auto) left( derestrict( *this ) );
4577  size_t i( 0UL );
4578 
4579  for( const auto& rowList : list ) {
4580  size_t j( 0UL );
4581  for( const auto& element : rowList ) {
4582  left(i,j) = element;
4583  ++j;
4584  }
4585  for( ; j<columns(); ++j ) {
4586  reset( left(i,j) );
4587  }
4588  ++i;
4589  }
4590 
4591  return *this;
4592 }
4594 //*************************************************************************************************
4595 
4596 
4597 //*************************************************************************************************
4612 template< typename MT // Type of the dense matrix
4613  , size_t... CSAs > // Compile time submatrix arguments
4614 inline Submatrix<MT,unaligned,true,true,CSAs...>&
4615  Submatrix<MT,unaligned,true,true,CSAs...>::operator=( const Submatrix& rhs )
4616 {
4619 
4620  if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
4621  return *this;
4622 
4623  if( rows() != rhs.rows() || columns() != rhs.columns() ) {
4624  BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
4625  }
4626 
4627  if( !tryAssign( matrix_, rhs, row(), column() ) ) {
4628  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4629  }
4630 
4631  decltype(auto) left( derestrict( *this ) );
4632 
4633  if( rhs.canAlias( &matrix_ ) ) {
4634  const ResultType tmp( rhs );
4635  smpAssign( left, tmp );
4636  }
4637  else {
4638  smpAssign( left, rhs );
4639  }
4640 
4641  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4642 
4643  return *this;
4644 }
4646 //*************************************************************************************************
4647 
4648 
4649 //*************************************************************************************************
4664 template< typename MT // Type of the dense matrix
4665  , size_t... CSAs > // Compile time submatrix arguments
4666 template< typename MT2 // Type of the right-hand side matrix
4667  , bool SO > // Storage order of the right-hand side matrix
4668 inline Submatrix<MT,unaligned,true,true,CSAs...>&
4669  Submatrix<MT,unaligned,true,true,CSAs...>::operator=( const Matrix<MT2,SO>& rhs )
4670 {
4672 
4673  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4674  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4675  }
4676 
4677  using Right = If_< IsRestricted<MT>, CompositeType_<MT2>, const MT2& >;
4678  Right right( ~rhs );
4679 
4680  if( !tryAssign( matrix_, right, row(), column() ) ) {
4681  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4682  }
4683 
4684  decltype(auto) left( derestrict( *this ) );
4685 
4686  if( IsReference<Right>::value && right.canAlias( &matrix_ ) ) {
4687  const ResultType_<MT2> tmp( right );
4688  if( IsSparseMatrix<MT2>::value )
4689  reset();
4690  smpAssign( left, tmp );
4691  }
4692  else {
4693  if( IsSparseMatrix<MT2>::value )
4694  reset();
4695  smpAssign( left, right );
4696  }
4697 
4698  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4699 
4700  return *this;
4701 }
4703 //*************************************************************************************************
4704 
4705 
4706 //*************************************************************************************************
4720 template< typename MT // Type of the dense matrix
4721  , size_t... CSAs > // Compile time submatrix arguments
4722 template< typename MT2 // Type of the right-hand side matrix
4723  , bool SO > // Storage order of the right-hand side matrix
4724 inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,unaligned,true,true,CSAs...>& >
4725  Submatrix<MT,unaligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
4726 {
4730 
4731  using AddType = AddTrait_< ResultType, ResultType_<MT2> >;
4732 
4735 
4736  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4737  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4738  }
4739 
4740  if( !tryAddAssign( matrix_, ~rhs, row(), column() ) ) {
4741  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4742  }
4743 
4744  decltype(auto) left( derestrict( *this ) );
4745 
4746  if( ( ( IsSymmetric<MT>::value || IsHermitian<MT>::value ) && hasOverlap() ) ||
4747  (~rhs).canAlias( &matrix_ ) ) {
4748  const AddType tmp( *this + (~rhs) );
4749  smpAssign( left, tmp );
4750  }
4751  else {
4752  smpAddAssign( left, ~rhs );
4753  }
4754 
4755  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4756 
4757  return *this;
4758 }
4760 //*************************************************************************************************
4761 
4762 
4763 //*************************************************************************************************
4777 template< typename MT // Type of the dense matrix
4778  , size_t... CSAs > // Compile time submatrix arguments
4779 template< typename MT2 // Type of the right-hand side matrix
4780  , bool SO > // Storage order of the right-hand side matrix
4781 inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,unaligned,true,true,CSAs...>& >
4782  Submatrix<MT,unaligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
4783 {
4787 
4788  using AddType = AddTrait_< ResultType, ResultType_<MT2> >;
4789 
4792 
4793  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4794  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4795  }
4796 
4797  const AddType tmp( *this + (~rhs) );
4798 
4799  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
4800  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4801  }
4802 
4803  decltype(auto) left( derestrict( *this ) );
4804 
4805  smpAssign( left, tmp );
4806 
4807  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4808 
4809  return *this;
4810 }
4812 //*************************************************************************************************
4813 
4814 
4815 //*************************************************************************************************
4829 template< typename MT // Type of the dense matrix
4830  , size_t... CSAs > // Compile time submatrix arguments
4831 template< typename MT2 // Type of the right-hand side matrix
4832  , bool SO > // Storage order of the right-hand side matrix
4833 inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,unaligned,true,true,CSAs...>& >
4834  Submatrix<MT,unaligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
4835 {
4839 
4840  using SubType = SubTrait_< ResultType, ResultType_<MT2> >;
4841 
4844 
4845  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4846  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4847  }
4848 
4849  if( !trySubAssign( matrix_, ~rhs, row(), column() ) ) {
4850  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4851  }
4852 
4853  decltype(auto) left( derestrict( *this ) );
4854 
4855  if( ( ( IsSymmetric<MT>::value || IsHermitian<MT>::value ) && hasOverlap() ) ||
4856  (~rhs).canAlias( &matrix_ ) ) {
4857  const SubType tmp( *this - (~rhs ) );
4858  smpAssign( left, tmp );
4859  }
4860  else {
4861  smpSubAssign( left, ~rhs );
4862  }
4863 
4864  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4865 
4866  return *this;
4867 }
4869 //*************************************************************************************************
4870 
4871 
4872 //*************************************************************************************************
4886 template< typename MT // Type of the dense matrix
4887  , size_t... CSAs > // Compile time submatrix arguments
4888 template< typename MT2 // Type of the right-hand side matrix
4889  , bool SO > // Storage order of the right-hand side matrix
4890 inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,unaligned,true,true,CSAs...>& >
4891  Submatrix<MT,unaligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
4892 {
4896 
4897  using SubType = SubTrait_< ResultType, ResultType_<MT2> >;
4898 
4901 
4902  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4903  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4904  }
4905 
4906  const SubType tmp( *this - (~rhs) );
4907 
4908  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
4909  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4910  }
4911 
4912  decltype(auto) left( derestrict( *this ) );
4913 
4914  smpAssign( left, tmp );
4915 
4916  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4917 
4918  return *this;
4919 }
4921 //*************************************************************************************************
4922 
4923 
4924 //*************************************************************************************************
4938 template< typename MT // Type of the dense matrix
4939  , size_t... CSAs > // Compile time submatrix arguments
4940 template< typename MT2 // Type of the right-hand side matrix
4941  , bool SO > // Storage order of the right-hand side matrix
4942 inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,unaligned,true,true,CSAs...>& >
4943  Submatrix<MT,unaligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
4944 {
4948 
4949  using SchurType = SchurTrait_< ResultType, ResultType_<MT2> >;
4950 
4952 
4953  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4954  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4955  }
4956 
4957  if( !trySchurAssign( matrix_, ~rhs, row(), column() ) ) {
4958  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4959  }
4960 
4961  decltype(auto) left( derestrict( *this ) );
4962 
4963  if( ( ( IsSymmetric<MT>::value || IsHermitian<MT>::value ) && hasOverlap() ) ||
4964  (~rhs).canAlias( &matrix_ ) ) {
4965  const SchurType tmp( *this % (~rhs) );
4966  if( IsSparseMatrix<SchurType>::value )
4967  reset();
4968  smpAssign( left, tmp );
4969  }
4970  else {
4971  smpSchurAssign( left, ~rhs );
4972  }
4973 
4974  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4975 
4976  return *this;
4977 }
4979 //*************************************************************************************************
4980 
4981 
4982 //*************************************************************************************************
4996 template< typename MT // Type of the dense matrix
4997  , size_t... CSAs > // Compile time submatrix arguments
4998 template< typename MT2 // Type of the right-hand side matrix
4999  , bool SO > // Storage order of the right-hand side matrix
5000 inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,unaligned,true,true,CSAs...>& >
5001  Submatrix<MT,unaligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
5002 {
5006 
5007  using SchurType = SchurTrait_< ResultType, ResultType_<MT2> >;
5008 
5010 
5011  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
5012  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
5013  }
5014 
5015  const SchurType tmp( *this % (~rhs) );
5016 
5017  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
5018  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
5019  }
5020 
5021  decltype(auto) left( derestrict( *this ) );
5022 
5023  if( IsSparseMatrix<SchurType>::value ) {
5024  reset();
5025  }
5026 
5027  smpAssign( left, tmp );
5028 
5029  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
5030 
5031  return *this;
5032 }
5034 //*************************************************************************************************
5035 
5036 
5037 
5038 
5039 //=================================================================================================
5040 //
5041 // UTILITY FUNCTIONS
5042 //
5043 //=================================================================================================
5044 
5045 //*************************************************************************************************
5051 template< typename MT // Type of the dense matrix
5052  , size_t... CSAs > // Compile time submatrix arguments
5053 inline MT& Submatrix<MT,unaligned,true,true,CSAs...>::operand() noexcept
5054 {
5055  return matrix_;
5056 }
5058 //*************************************************************************************************
5059 
5060 
5061 //*************************************************************************************************
5067 template< typename MT // Type of the dense matrix
5068  , size_t... CSAs > // Compile time submatrix arguments
5069 inline const MT& Submatrix<MT,unaligned,true,true,CSAs...>::operand() const noexcept
5070 {
5071  return matrix_;
5072 }
5074 //*************************************************************************************************
5075 
5076 
5077 //*************************************************************************************************
5086 template< typename MT // Type of the dense matrix
5087  , size_t... CSAs > // Compile time submatrix arguments
5088 inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::spacing() const noexcept
5089 {
5090  return matrix_.spacing();
5091 }
5093 //*************************************************************************************************
5094 
5095 
5096 //*************************************************************************************************
5102 template< typename MT // Type of the dense matrix
5103  , size_t... CSAs > // Compile time submatrix arguments
5104 inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::capacity() const noexcept
5105 {
5106  return rows() * columns();
5107 }
5109 //*************************************************************************************************
5110 
5111 
5112 //*************************************************************************************************
5119 template< typename MT // Type of the dense matrix
5120  , size_t... CSAs > // Compile time submatrix arguments
5121 inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::capacity( size_t j ) const noexcept
5122 {
5123  UNUSED_PARAMETER( j );
5124 
5125  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
5126 
5127  return rows();
5128 }
5130 //*************************************************************************************************
5131 
5132 
5133 //*************************************************************************************************
5139 template< typename MT // Type of the dense matrix
5140  , size_t... CSAs > // Compile time submatrix arguments
5142 {
5143  const size_t iend( row() + rows() );
5144  const size_t jend( column() + columns() );
5145  size_t nonzeros( 0UL );
5146 
5147  for( size_t j=column(); j<jend; ++j )
5148  for( size_t i=row(); i<iend; ++i )
5149  if( !isDefault( matrix_(i,j) ) )
5150  ++nonzeros;
5151 
5152  return nonzeros;
5153 }
5155 //*************************************************************************************************
5156 
5157 
5158 //*************************************************************************************************
5165 template< typename MT // Type of the dense matrix
5166  , size_t... CSAs > // Compile time submatrix arguments
5167 inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::nonZeros( size_t j ) const
5168 {
5169  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
5170 
5171  const size_t iend( row() + rows() );
5172  size_t nonzeros( 0UL );
5173 
5174  for( size_t i=row(); i<iend; ++i )
5175  if( !isDefault( matrix_(i,column()+j) ) )
5176  ++nonzeros;
5177 
5178  return nonzeros;
5179 }
5181 //*************************************************************************************************
5182 
5183 
5184 //*************************************************************************************************
5190 template< typename MT // Type of the dense matrix
5191  , size_t... CSAs > // Compile time submatrix arguments
5193 {
5194  using blaze::clear;
5195 
5196  for( size_t j=column(); j<column()+columns(); ++j )
5197  {
5198  const size_t ibegin( ( IsLower<MT>::value )
5199  ?( ( IsUniLower<MT>::value || IsStrictlyLower<MT>::value )
5200  ?( max( j+1UL, row() ) )
5201  :( max( j, row() ) ) )
5202  :( row() ) );
5203  const size_t iend ( ( IsUpper<MT>::value )
5204  ?( ( IsUniUpper<MT>::value || IsStrictlyUpper<MT>::value )
5205  ?( min( j, row()+rows() ) )
5206  :( min( j+1UL, row()+rows() ) ) )
5207  :( row()+rows() ) );
5208 
5209  for( size_t i=ibegin; i<iend; ++i )
5210  clear( matrix_(i,j) );
5211  }
5212 }
5214 //*************************************************************************************************
5215 
5216 
5217 //*************************************************************************************************
5224 template< typename MT // Type of the dense matrix
5225  , size_t... CSAs > // Compile time submatrix arguments
5227 {
5228  using blaze::clear;
5229 
5230  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
5231 
5232  const size_t ibegin( ( IsLower<MT>::value )
5233  ?( ( IsUniLower<MT>::value || IsStrictlyLower<MT>::value )
5234  ?( max( j+1UL, row() ) )
5235  :( max( j, row() ) ) )
5236  :( row() ) );
5237  const size_t iend ( ( IsUpper<MT>::value )
5238  ?( ( IsUniUpper<MT>::value || IsStrictlyUpper<MT>::value )
5239  ?( min( j, row()+rows() ) )
5240  :( min( j+1UL, row()+rows() ) ) )
5241  :( row()+rows() ) );
5242 
5243  for( size_t i=ibegin; i<iend; ++i )
5244  clear( matrix_(i,column()+j) );
5245 }
5247 //*************************************************************************************************
5248 
5249 
5250 //*************************************************************************************************
5260 template< typename MT // Type of the dense matrix
5261  , size_t... CSAs > // Compile time submatrix arguments
5262 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::hasOverlap() const noexcept
5263 {
5264  BLAZE_INTERNAL_ASSERT( IsSymmetric<MT>::value || IsHermitian<MT>::value, "Invalid matrix detected" );
5265 
5266  if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
5267  return false;
5268  else return true;
5269 }
5271 //*************************************************************************************************
5272 
5273 
5274 
5275 
5276 //=================================================================================================
5277 //
5278 // NUMERIC FUNCTIONS
5279 //
5280 //=================================================================================================
5281 
5282 //*************************************************************************************************
5300 template< typename MT // Type of the dense matrix
5301  , size_t... CSAs > // Compile time submatrix arguments
5302 inline Submatrix<MT,unaligned,true,true,CSAs...>&
5304 {
5305  if( rows() != columns() ) {
5306  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
5307  }
5308 
5309  if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
5310  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
5311  }
5312 
5313  decltype(auto) left( derestrict( *this ) );
5314  const ResultType tmp( trans( *this ) );
5315 
5316  smpAssign( left, tmp );
5317 
5318  return *this;
5319 }
5321 //*************************************************************************************************
5322 
5323 
5324 //*************************************************************************************************
5342 template< typename MT // Type of the dense matrix
5343  , size_t... CSAs > // Compile time submatrix arguments
5344 inline Submatrix<MT,unaligned,true,true,CSAs...>&
5345  Submatrix<MT,unaligned,true,true,CSAs...>::ctranspose()
5346 {
5347  if( rows() != columns() ) {
5348  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
5349  }
5350 
5351  if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
5352  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
5353  }
5354 
5355  decltype(auto) left( derestrict( *this ) );
5356  const ResultType tmp( ctrans( *this ) );
5357 
5358  smpAssign( left, tmp );
5359 
5360  return *this;
5361 }
5363 //*************************************************************************************************
5364 
5365 
5366 //*************************************************************************************************
5379 template< typename MT // Type of the dense matrix
5380  , size_t... CSAs > // Compile time submatrix arguments
5381 template< typename Other > // Data type of the scalar value
5382 inline Submatrix<MT,unaligned,true,true,CSAs...>&
5383  Submatrix<MT,unaligned,true,true,CSAs...>::scale( const Other& scalar )
5384 {
5386 
5387  const size_t jend( column() + columns() );
5388 
5389  for( size_t j=column(); j<jend; ++j )
5390  {
5391  const size_t ibegin( ( IsLower<MT>::value )
5392  ?( ( IsStrictlyLower<MT>::value )
5393  ?( max( j+1UL, row() ) )
5394  :( max( j, row() ) ) )
5395  :( row() ) );
5396  const size_t iend ( ( IsUpper<MT>::value )
5397  ?( ( IsStrictlyUpper<MT>::value )
5398  ?( min( j, row()+rows() ) )
5399  :( min( j+1UL, row()+rows() ) ) )
5400  :( row()+rows() ) );
5401 
5402  for( size_t i=ibegin; i<iend; ++i )
5403  matrix_(i,j) *= scalar;
5404  }
5405 
5406  return *this;
5407 }
5409 //*************************************************************************************************
5410 
5411 
5412 
5413 
5414 //=================================================================================================
5415 //
5416 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
5417 //
5418 //=================================================================================================
5419 
5420 //*************************************************************************************************
5431 template< typename MT // Type of the dense matrix
5432  , size_t... CSAs > // Compile time submatrix arguments
5433 template< typename Other > // Data type of the foreign expression
5434 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::canAlias( const Other* alias ) const noexcept
5435 {
5436  return matrix_.isAliased( alias );
5437 }
5439 //*************************************************************************************************
5440 
5441 
5442 //*************************************************************************************************
5453 template< typename MT // Type of the dense matrix
5454  , size_t... CSAs > // Compile time submatrix arguments
5455 template< typename MT2 // Data type of the foreign dense submatrix
5456  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
5457  , bool SO2 // Storage order of the foreign dense submatrix
5458  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
5459 inline bool
5460  Submatrix<MT,unaligned,true,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
5461 {
5462  return ( matrix_.isAliased( &alias->matrix_ ) &&
5463  ( row() + rows() > alias->row() ) &&
5464  ( row() < alias->row() + alias->rows() ) &&
5465  ( column() + columns() > alias->column() ) &&
5466  ( column() < alias->column() + alias->columns() ) );
5467 }
5469 //*************************************************************************************************
5470 
5471 
5472 //*************************************************************************************************
5483 template< typename MT // Type of the dense matrix
5484  , size_t... CSAs > // Compile time submatrix arguments
5485 template< typename Other > // Data type of the foreign expression
5486 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::isAliased( const Other* alias ) const noexcept
5487 {
5488  return matrix_.isAliased( alias );
5489 }
5491 //*************************************************************************************************
5492 
5493 
5494 //*************************************************************************************************
5505 template< typename MT // Type of the dense matrix
5506  , size_t... CSAs > // Compile time submatrix arguments
5507 template< typename MT2 // Data type of the foreign dense submatrix
5508  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
5509  , bool SO2 // Storage order of the foreign dense submatrix
5510  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
5511 inline bool
5512  Submatrix<MT,unaligned,true,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
5513 {
5514  return ( matrix_.isAliased( &alias->matrix_ ) &&
5515  ( row() + rows() > alias->row() ) &&
5516  ( row() < alias->row() + alias->rows() ) &&
5517  ( column() + columns() > alias->column() ) &&
5518  ( column() < alias->column() + alias->columns() ) );
5519 }
5521 //*************************************************************************************************
5522 
5523 
5524 //*************************************************************************************************
5534 template< typename MT // Type of the dense matrix
5535  , size_t... CSAs > // Compile time submatrix arguments
5536 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::isAligned() const noexcept
5537 {
5538  return isAligned_;
5539 }
5541 //*************************************************************************************************
5542 
5543 
5544 //*************************************************************************************************
5555 template< typename MT // Type of the dense matrix
5556  , size_t... CSAs > // Compile time submatrix arguments
5557 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::canSMPAssign() const noexcept
5558 {
5559  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
5560 }
5562 //*************************************************************************************************
5563 
5564 
5565 //*************************************************************************************************
5580 template< typename MT // Type of the dense matrix
5581  , size_t... CSAs > // Compile time submatrix arguments
5582 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,true,true,CSAs...>::SIMDType
5583  Submatrix<MT,unaligned,true,true,CSAs...>::load( size_t i, size_t j ) const noexcept
5584 {
5585  if( isAligned_ )
5586  return loada( i, j );
5587  else
5588  return loadu( i, j );
5589 }
5591 //*************************************************************************************************
5592 
5593 
5594 //*************************************************************************************************
5609 template< typename MT // Type of the dense matrix
5610  , size_t... CSAs > // Compile time submatrix arguments
5611 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,true,true,CSAs...>::SIMDType
5612  Submatrix<MT,unaligned,true,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
5613 {
5615 
5616  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5617  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5618  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5619  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5620 
5621  return matrix_.loada( row()+i, column()+j );
5622 }
5624 //*************************************************************************************************
5625 
5626 
5627 //*************************************************************************************************
5642 template< typename MT // Type of the dense matrix
5643  , size_t... CSAs > // Compile time submatrix arguments
5644 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,true,true,CSAs...>::SIMDType
5645  Submatrix<MT,unaligned,true,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
5646 {
5648 
5649  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5650  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5651  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5652  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5653 
5654  return matrix_.loadu( row()+i, column()+j );
5655 }
5657 //*************************************************************************************************
5658 
5659 
5660 //*************************************************************************************************
5676 template< typename MT // Type of the dense matrix
5677  , size_t... CSAs > // Compile time submatrix arguments
5679  Submatrix<MT,unaligned,true,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
5680 {
5681  if( isAligned_ )
5682  storea( i, j, value );
5683  else
5684  storeu( i, j, value );
5685 }
5687 //*************************************************************************************************
5688 
5689 
5690 //*************************************************************************************************
5706 template< typename MT // Type of the dense matrix
5707  , size_t... CSAs > // Compile time submatrix arguments
5709  Submatrix<MT,unaligned,true,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
5710 {
5712 
5713  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5714  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5715  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5716  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5717 
5718  matrix_.storea( row()+i, column()+j, value );
5719 }
5721 //*************************************************************************************************
5722 
5723 
5724 //*************************************************************************************************
5740 template< typename MT // Type of the dense matrix
5741  , size_t... CSAs > // Compile time submatrix arguments
5743  Submatrix<MT,unaligned,true,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
5744 {
5746 
5747  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5748  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5749  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5750  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5751 
5752  matrix_.storeu( row()+i, column()+j, value );
5753 }
5755 //*************************************************************************************************
5756 
5757 
5758 //*************************************************************************************************
5775 template< typename MT // Type of the dense matrix
5776  , size_t... CSAs > // Compile time submatrix arguments
5778  Submatrix<MT,unaligned,true,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
5779 {
5781 
5782  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5783  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5784  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5785  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5786 
5787  if( isAligned_ )
5788  matrix_.stream( row()+i, column()+j, value );
5789  else
5790  matrix_.storeu( row()+i, column()+j, value );
5791 }
5793 //*************************************************************************************************
5794 
5795 
5796 //*************************************************************************************************
5808 template< typename MT // Type of the dense matrix
5809  , size_t... CSAs > // Compile time submatrix arguments
5810 template< typename MT2 > // Type of the right-hand side dense matrix
5811 inline DisableIf_< typename Submatrix<MT,unaligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedAssign<MT2> >
5812  Submatrix<MT,unaligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
5813 {
5814  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5815  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5816 
5817  const size_t ipos( rows() & size_t(-2) );
5818  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
5819 
5820  for( size_t j=0UL; j<columns(); ++j ) {
5821  for( size_t i=0UL; i<ipos; i+=2UL ) {
5822  matrix_(row()+i ,column()+j) = (~rhs)(i ,j);
5823  matrix_(row()+i+1UL,column()+j) = (~rhs)(i+1UL,j);
5824  }
5825  if( ipos < rows() ) {
5826  matrix_(row()+ipos,column()+j) = (~rhs)(ipos,j);
5827  }
5828  }
5829 }
5831 //*************************************************************************************************
5832 
5833 
5834 //*************************************************************************************************
5846 template< typename MT // Type of the dense matrix
5847  , size_t... CSAs > // Compile time submatrix arguments
5848 template< typename MT2 > // Type of the right-hand side dense matrix
5849 inline EnableIf_< typename Submatrix<MT,unaligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedAssign<MT2> >
5850  Submatrix<MT,unaligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
5851 {
5853 
5854  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5855  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5856 
5857  const size_t ipos( rows() & size_t(-SIMDSIZE) );
5858  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
5859 
5860  if( useStreaming && isAligned_ &&
5861  rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
5862  !(~rhs).isAliased( &matrix_ ) )
5863  {
5864  for( size_t j=0UL; j<columns(); ++j )
5865  {
5866  size_t i( 0UL );
5867  Iterator left( begin(j) );
5868  ConstIterator_<MT2> right( (~rhs).begin(j) );
5869 
5870  for( ; i<ipos; i+=SIMDSIZE ) {
5871  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5872  }
5873  for( ; i<rows(); ++i ) {
5874  *left = *right; ++left; ++right;
5875  }
5876  }
5877  }
5878  else
5879  {
5880  for( size_t j=0UL; j<columns(); ++j )
5881  {
5882  size_t i( 0UL );
5883  Iterator left( begin(j) );
5884  ConstIterator_<MT2> right( (~rhs).begin(j) );
5885 
5886  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
5887  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5888  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5889  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5890  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5891  }
5892  for( ; i<ipos; i+=SIMDSIZE ) {
5893  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5894  }
5895  for( ; i<rows(); ++i ) {
5896  *left = *right; ++left; ++right;
5897  }
5898  }
5899  }
5900 }
5902 //*************************************************************************************************
5903 
5904 
5905 //*************************************************************************************************
5917 template< typename MT // Type of the dense matrix
5918  , size_t... CSAs > // Compile time submatrix arguments
5919 template< typename MT2 > // Type of the right-hand side dense matrix
5920 inline void Submatrix<MT,unaligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
5921 {
5923 
5924  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5925  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5926 
5927  constexpr size_t block( BLOCK_SIZE );
5928 
5929  for( size_t jj=0UL; jj<columns(); jj+=block ) {
5930  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
5931  for( size_t ii=0UL; ii<rows(); ii+=block ) {
5932  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
5933  for( size_t j=jj; j<jend; ++j ) {
5934  for( size_t i=ii; i<iend; ++i ) {
5935  matrix_(row()+i,column()+j) = (~rhs)(i,j);
5936  }
5937  }
5938  }
5939  }
5940 }
5942 //*************************************************************************************************
5943 
5944 
5945 //*************************************************************************************************
5957 template< typename MT // Type of the dense matrix
5958  , size_t... CSAs > // Compile time submatrix arguments
5959 template< typename MT2 > // Type of the right-hand side sparse matrix
5960 inline void Submatrix<MT,unaligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
5961 {
5962  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5963  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5964 
5965  for( size_t j=0UL; j<columns(); ++j )
5966  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
5967  matrix_(row()+element->index(),column()+j) = element->value();
5968 }
5970 //*************************************************************************************************
5971 
5972 
5973 //*************************************************************************************************
5985 template< typename MT // Type of the dense matrix
5986  , size_t... CSAs > // Compile time submatrix arguments
5987 template< typename MT2 > // Type of the right-hand side sparse matrix
5988 inline void Submatrix<MT,unaligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
5989 {
5991 
5992  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5993  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5994 
5995  for( size_t i=0UL; i<rows(); ++i )
5996  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
5997  matrix_(row()+i,column()+element->index()) = element->value();
5998 }
6000 //*************************************************************************************************
6001 
6002 
6003 //*************************************************************************************************
6015 template< typename MT // Type of the dense matrix
6016  , size_t... CSAs > // Compile time submatrix arguments
6017 template< typename MT2 > // Type of the right-hand side dense matrix
6018 inline DisableIf_< typename Submatrix<MT,unaligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedAddAssign<MT2> >
6019  Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
6020 {
6021  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6022  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6023 
6024  const size_t ipos( rows() & size_t(-2) );
6025  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
6026 
6027  for( size_t j=0UL; j<columns(); ++j )
6028  {
6029  if( IsDiagonal<MT2>::value ) {
6030  matrix_(row()+j,column()+j) += (~rhs)(j,j);
6031  }
6032  else {
6033  for( size_t i=0UL; i<ipos; i+=2UL ) {
6034  matrix_(row()+i ,column()+j) += (~rhs)(i ,j);
6035  matrix_(row()+i+1UL,column()+j) += (~rhs)(i+1UL,j);
6036  }
6037  if( ipos < rows() ) {
6038  matrix_(row()+ipos,column()+j) += (~rhs)(ipos,j);
6039  }
6040  }
6041  }
6042 }
6044 //*************************************************************************************************
6045 
6046 
6047 //*************************************************************************************************
6059 template< typename MT // Type of the dense matrix
6060  , size_t... CSAs > // Compile time submatrix arguments
6061 template< typename MT2 > // Type of the right-hand side dense matrix
6062 inline EnableIf_< typename Submatrix<MT,unaligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedAddAssign<MT2> >
6063  Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
6064 {
6066 
6067  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6068  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6069 
6070  for( size_t j=0UL; j<columns(); ++j )
6071  {
6072  const size_t ibegin( ( IsLower<MT>::value )
6073  ?( ( IsStrictlyLower<MT>::value ? j+1UL : j ) & size_t(-SIMDSIZE) )
6074  :( 0UL ) );
6075  const size_t iend ( ( IsUpper<MT>::value )
6076  ?( IsStrictlyUpper<MT>::value ? j : j+1UL )
6077  :( rows() ) );
6078  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
6079 
6080  const size_t ipos( iend & size_t(-SIMDSIZE) );
6081  BLAZE_INTERNAL_ASSERT( ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
6082 
6083  size_t i( ibegin );
6084  Iterator left( begin(j) + ibegin );
6085  ConstIterator_<MT2> right( (~rhs).begin(j) + ibegin );
6086 
6087  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6088  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6089  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6090  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6091  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6092  }
6093  for( ; i<ipos; i+=SIMDSIZE ) {
6094  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6095  }
6096  for( ; i<iend; ++i ) {
6097  *left += *right; ++left; ++right;
6098  }
6099  }
6100 }
6102 //*************************************************************************************************
6103 
6104 
6105 //*************************************************************************************************
6117 template< typename MT // Type of the dense matrix
6118  , size_t... CSAs > // Compile time submatrix arguments
6119 template< typename MT2 > // Type of the right-hand side dense matrix
6120 inline void Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
6121 {
6123 
6124  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6125  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6126 
6127  constexpr size_t block( BLOCK_SIZE );
6128 
6129  for( size_t jj=0UL; jj<columns(); jj+=block ) {
6130  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
6131  for( size_t ii=0UL; ii<rows(); ii+=block ) {
6132  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
6133  for( size_t j=jj; j<jend; ++j ) {
6134  for( size_t i=ii; i<iend; ++i ) {
6135  matrix_(row()+i,column()+j) += (~rhs)(i,j);
6136  }
6137  }
6138  }
6139  }
6140 }
6142 //*************************************************************************************************
6143 
6144 
6145 //*************************************************************************************************
6157 template< typename MT // Type of the dense matrix
6158  , size_t... CSAs > // Compile time submatrix arguments
6159 template< typename MT2 > // Type of the right-hand side sparse matrix
6160 inline void Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
6161 {
6162  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6163  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6164 
6165  for( size_t j=0UL; j<columns(); ++j )
6166  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
6167  matrix_(row()+element->index(),column()+j) += element->value();
6168 }
6170 //*************************************************************************************************
6171 
6172 
6173 //*************************************************************************************************
6185 template< typename MT // Type of the dense matrix
6186  , size_t... CSAs > // Compile time submatrix arguments
6187 template< typename MT2 > // Type of the right-hand side sparse matrix
6188 inline void Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
6189 {
6191 
6192  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6193  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6194 
6195  for( size_t i=0UL; i<rows(); ++i )
6196  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
6197  matrix_(row()+i,column()+element->index()) += element->value();
6198 }
6200 //*************************************************************************************************
6201 
6202 
6203 //*************************************************************************************************
6215 template< typename MT // Type of the dense matrix
6216  , size_t... CSAs > // Compile time submatrix arguments
6217 template< typename MT2 > // Type of the right-hand side dense matrix
6218 inline DisableIf_< typename Submatrix<MT,unaligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedSubAssign<MT2> >
6219  Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
6220 {
6221  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6222  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6223 
6224  const size_t ipos( rows() & size_t(-2) );
6225  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
6226 
6227  for( size_t j=0UL; j<columns(); ++j )
6228  {
6229  if( IsDiagonal<MT2>::value ) {
6230  matrix_(row()+j,column()+j) -= (~rhs)(j,j);
6231  }
6232  else {
6233  for( size_t i=0UL; i<ipos; i+=2UL ) {
6234  matrix_(row()+i ,column()+j) -= (~rhs)(i ,j);
6235  matrix_(row()+i+1UL,column()+j) -= (~rhs)(i+1UL,j);
6236  }
6237  if( ipos < rows() ) {
6238  matrix_(row()+ipos,column()+j) -= (~rhs)(ipos,j);
6239  }
6240  }
6241  }
6242 }
6244 //*************************************************************************************************
6245 
6246 
6247 //*************************************************************************************************
6259 template< typename MT // Type of the dense matrix
6260  , size_t... CSAs > // Compile time submatrix arguments
6261 template< typename MT2 > // Type of the right-hand side dense matrix
6262 inline EnableIf_< typename Submatrix<MT,unaligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedSubAssign<MT2> >
6263  Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
6264 {
6266 
6267  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6268  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6269 
6270  for( size_t j=0UL; j<columns(); ++j )
6271  {
6272  const size_t ibegin( ( IsLower<MT>::value )
6273  ?( ( IsStrictlyLower<MT>::value ? j+1UL : j ) & size_t(-SIMDSIZE) )
6274  :( 0UL ) );
6275  const size_t iend ( ( IsUpper<MT>::value )
6276  ?( IsStrictlyUpper<MT>::value ? j : j+1UL )
6277  :( rows() ) );
6278  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
6279 
6280  const size_t ipos( iend & size_t(-SIMDSIZE) );
6281  BLAZE_INTERNAL_ASSERT( ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
6282 
6283  size_t i( ibegin );
6284  Iterator left( begin(j) + ibegin );
6285  ConstIterator_<MT2> right( (~rhs).begin(j) + ibegin );
6286 
6287  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6288  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6289  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6290  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6291  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6292  }
6293  for( ; i<ipos; i+=SIMDSIZE ) {
6294  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6295  }
6296  for( ; i<iend; ++i ) {
6297  *left -= *right; ++left; ++right;
6298  }
6299  }
6300 }
6302 //*************************************************************************************************
6303 
6304 
6305 //*************************************************************************************************
6317 template< typename MT // Type of the dense matrix
6318  , size_t... CSAs > // Compile time submatrix arguments
6319 template< typename MT2 > // Type of the right-hand side dense matrix
6320 inline void Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
6321 {
6323 
6324  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6325  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6326 
6327  constexpr size_t block( BLOCK_SIZE );
6328 
6329  for( size_t jj=0UL; jj<columns(); jj+=block ) {
6330  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
6331  for( size_t ii=0UL; ii<rows(); ii+=block ) {
6332  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
6333  for( size_t j=jj; j<jend; ++j ) {
6334  for( size_t i=ii; i<iend; ++i ) {
6335  matrix_(row()+i,column()+j) -= (~rhs)(i,j);
6336  }
6337  }
6338  }
6339  }
6340 }
6342 //*************************************************************************************************
6343 
6344 
6345 //*************************************************************************************************
6357 template< typename MT // Type of the dense matrix
6358  , size_t... CSAs > // Compile time submatrix arguments
6359 template< typename MT2 > // Type of the right-hand side sparse matrix
6360 inline void Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
6361 {
6362  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6363  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6364 
6365  for( size_t j=0UL; j<columns(); ++j )
6366  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
6367  matrix_(row()+element->index(),column()+j) -= element->value();
6368 }
6370 //*************************************************************************************************
6371 
6372 
6373 //*************************************************************************************************
6385 template< typename MT // Type of the dense matrix
6386  , size_t... CSAs > // Compile time submatrix arguments
6387 template< typename MT2 > // Type of the right-hand side sparse matrix
6388 inline void Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
6389 {
6391 
6392  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6393  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6394 
6395  for( size_t i=0UL; i<rows(); ++i )
6396  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
6397  matrix_(row()+i,column()+element->index()) -= element->value();
6398 }
6400 //*************************************************************************************************
6401 
6402 
6403 //*************************************************************************************************
6415 template< typename MT // Type of the dense matrix
6416  , size_t... CSAs > // Compile time submatrix arguments
6417 template< typename MT2 > // Type of the right-hand side dense matrix
6418 inline DisableIf_< typename Submatrix<MT,unaligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedSchurAssign<MT2> >
6419  Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
6420 {
6421  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6422  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6423 
6424  const size_t ipos( rows() & size_t(-2) );
6425  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
6426 
6427  for( size_t j=0UL; j<columns(); ++j ) {
6428  for( size_t i=0UL; i<ipos; i+=2UL ) {
6429  matrix_(row()+i ,column()+j) *= (~rhs)(i ,j);
6430  matrix_(row()+i+1UL,column()+j) *= (~rhs)(i+1UL,j);
6431  }
6432  if( ipos < rows() ) {
6433  matrix_(row()+ipos,column()+j) *= (~rhs)(ipos,j);
6434  }
6435  }
6436 }
6438 //*************************************************************************************************
6439 
6440 
6441 //*************************************************************************************************
6454 template< typename MT // Type of the dense matrix
6455  , size_t... CSAs > // Compile time submatrix arguments
6456 template< typename MT2 > // Type of the right-hand side dense matrix
6457 inline EnableIf_< typename Submatrix<MT,unaligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedSchurAssign<MT2> >
6458  Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
6459 {
6461 
6462  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6463  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6464 
6465  for( size_t j=0UL; j<columns(); ++j )
6466  {
6467  const size_t ipos( rows() & size_t(-SIMDSIZE) );
6468  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
6469 
6470  size_t i( 0UL );
6471  Iterator left( begin(j) );
6472  ConstIterator_<MT2> right( (~rhs).begin(j) );
6473 
6474  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6475  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6476  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6477  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6478  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6479  }
6480  for( ; i<ipos; i+=SIMDSIZE ) {
6481  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6482  }
6483  for( ; i<rows(); ++i ) {
6484  *left *= *right; ++left; ++right;
6485  }
6486  }
6487 }
6489 //*************************************************************************************************
6490 
6491 
6492 //*************************************************************************************************
6504 template< typename MT // Type of the dense matrix
6505  , size_t... CSAs > // Compile time submatrix arguments
6506 template< typename MT2 > // Type of the right-hand side dense matrix
6507 inline void Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
6508 {
6510 
6511  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6512  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6513 
6514  constexpr size_t block( BLOCK_SIZE );
6515 
6516  for( size_t jj=0UL; jj<columns(); jj+=block ) {
6517  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
6518  for( size_t ii=0UL; ii<rows(); ii+=block ) {
6519  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
6520  for( size_t j=jj; j<jend; ++j ) {
6521  for( size_t i=ii; i<iend; ++i ) {
6522  matrix_(row()+i,column()+j) *= (~rhs)(i,j);
6523  }
6524  }
6525  }
6526  }
6527 }
6529 //*************************************************************************************************
6530 
6531 
6532 //*************************************************************************************************
6544 template< typename MT // Type of the dense matrix
6545  , size_t... CSAs > // Compile time submatrix arguments
6546 template< typename MT2 > // Type of the right-hand side sparse matrix
6547 inline void Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
6548 {
6549  using blaze::reset;
6550 
6551  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6552  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6553 
6554  for( size_t j=0UL; j<columns(); ++j )
6555  {
6556  size_t i( 0UL );
6557 
6558  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element ) {
6559  for( ; i<element->index(); ++i )
6560  reset( matrix_(row()+i,column()+j) );
6561  matrix_(row()+i,column()+j) *= element->value();
6562  ++i;
6563  }
6564 
6565  for( ; i<rows(); ++i ) {
6566  reset( matrix_(row()+i,column()+j) );
6567  }
6568  }
6569 }
6571 //*************************************************************************************************
6572 
6573 
6574 //*************************************************************************************************
6586 template< typename MT // Type of the dense matrix
6587  , size_t... CSAs > // Compile time submatrix arguments
6588 template< typename MT2 > // Type of the right-hand side sparse matrix
6589 inline void Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
6590 {
6591  using blaze::reset;
6592 
6594 
6595  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6596  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6597 
6598  for( size_t i=0UL; i<rows(); ++i )
6599  {
6600  size_t j( 0UL );
6601 
6602  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element ) {
6603  for( ; j<element->index(); ++j )
6604  reset( matrix_(row()+i,column()+j) );
6605  matrix_(row()+i,column()+j) *= element->value();
6606  ++j;
6607  }
6608 
6609  for( ; j<columns(); ++j ) {
6610  reset( matrix_(row()+i,column()+j) );
6611  }
6612  }
6613 }
6615 //*************************************************************************************************
6616 
6617 
6618 
6619 
6620 
6621 
6622 
6623 
6624 //=================================================================================================
6625 //
6626 // CLASS TEMPLATE SPECIALIZATION FOR ALIGNED ROW-MAJOR DENSE SUBMATRICES
6627 //
6628 //=================================================================================================
6629 
6630 //*************************************************************************************************
6638 template< typename MT // Type of the dense matrix
6639  , size_t... CSAs > // Compile time submatrix arguments
6640 class Submatrix<MT,aligned,false,true,CSAs...>
6641  : public View< DenseMatrix< Submatrix<MT,aligned,false,true,CSAs...>, false > >
6642  , private SubmatrixData<CSAs...>
6643 {
6644  private:
6645  //**Type definitions****************************************************************************
6646  using DataType = SubmatrixData<CSAs...>;
6647  using Operand = If_< IsExpression<MT>, MT, MT& >;
6648  //**********************************************************************************************
6649 
6650  public:
6651  //**Type definitions****************************************************************************
6653  using This = Submatrix<MT,aligned,false,true,CSAs...>;
6654 
6655  using BaseType = DenseMatrix<This,false>;
6656  using ViewedType = MT;
6657  using ResultType = SubmatrixTrait_<MT,CSAs...>;
6658  using OppositeType = OppositeType_<ResultType>;
6659  using TransposeType = TransposeType_<ResultType>;
6660  using ElementType = ElementType_<MT>;
6661  using SIMDType = SIMDTrait_<ElementType>;
6662  using ReturnType = ReturnType_<MT>;
6663  using CompositeType = const Submatrix&;
6664 
6666  using ConstReference = ConstReference_<MT>;
6667 
6669  using Reference = If_< IsConst<MT>, ConstReference, Reference_<MT> >;
6670 
6672  using ConstPointer = ConstPointer_<MT>;
6673 
6675  using Pointer = If_< Or< IsConst<MT>, Not< HasMutableDataAccess<MT> > >, ConstPointer, Pointer_<MT> >;
6676 
6678  using ConstIterator = ConstIterator_<MT>;
6679 
6681  using Iterator = If_< IsConst<MT>, ConstIterator, Iterator_<MT> >;
6682  //**********************************************************************************************
6683 
6684  //**Compilation flags***************************************************************************
6686  enum : bool { simdEnabled = MT::simdEnabled };
6687 
6689  enum : bool { smpAssignable = MT::smpAssignable };
6690  //**********************************************************************************************
6691 
6692  //**Constructors********************************************************************************
6695  template< typename... RSAs >
6696  explicit inline Submatrix( MT& matrix, RSAs... args );
6697  // No explicitly declared copy constructor.
6699  //**********************************************************************************************
6700 
6701  //**Destructor**********************************************************************************
6702  // No explicitly declared destructor.
6703  //**********************************************************************************************
6704 
6705  //**Data access functions***********************************************************************
6708  inline Reference operator()( size_t i, size_t j );
6709  inline ConstReference operator()( size_t i, size_t j ) const;
6710  inline Reference at( size_t i, size_t j );
6711  inline ConstReference at( size_t i, size_t j ) const;
6712  inline Pointer data () noexcept;
6713  inline ConstPointer data () const noexcept;
6714  inline Pointer data ( size_t i ) noexcept;
6715  inline ConstPointer data ( size_t i ) const noexcept;
6716  inline Iterator begin ( size_t i );
6717  inline ConstIterator begin ( size_t i ) const;
6718  inline ConstIterator cbegin( size_t i ) const;
6719  inline Iterator end ( size_t i );
6720  inline ConstIterator end ( size_t i ) const;
6721  inline ConstIterator cend ( size_t i ) const;
6723  //**********************************************************************************************
6724 
6725  //**Assignment operators************************************************************************
6728  inline Submatrix& operator=( const ElementType& rhs );
6729  inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
6730  inline Submatrix& operator=( const Submatrix& rhs );
6731 
6732  template< typename MT2, bool SO >
6733  inline Submatrix& operator=( const Matrix<MT2,SO>& rhs );
6734 
6735  template< typename MT2, bool SO >
6736  inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
6737  operator+=( const Matrix<MT2,SO>& rhs );
6738 
6739  template< typename MT2, bool SO >
6740  inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
6741  operator+=( const Matrix<MT2,SO>& rhs );
6742 
6743  template< typename MT2, bool SO >
6744  inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
6745  operator-=( const Matrix<MT2,SO>& rhs );
6746 
6747  template< typename MT2, bool SO >
6748  inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
6749  operator-=( const Matrix<MT2,SO>& rhs );
6750 
6751  template< typename MT2, bool SO >
6752  inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
6753  operator%=( const Matrix<MT2,SO>& rhs );
6754 
6755  template< typename MT2, bool SO >
6756  inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
6757  operator%=( const Matrix<MT2,SO>& rhs );
6759  //**********************************************************************************************
6760 
6761  //**Utility functions***************************************************************************
6764  using DataType::row;
6765  using DataType::column;
6766  using DataType::rows;
6767  using DataType::columns;
6768 
6769  inline MT& operand() noexcept;
6770  inline const MT& operand() const noexcept;
6771 
6772  inline size_t spacing() const noexcept;
6773  inline size_t capacity() const noexcept;
6774  inline size_t capacity( size_t i ) const noexcept;
6775  inline size_t nonZeros() const;
6776  inline size_t nonZeros( size_t i ) const;
6777  inline void reset();
6778  inline void reset( size_t i );
6780  //**********************************************************************************************
6781 
6782  //**Numeric functions***************************************************************************
6785  inline Submatrix& transpose();
6786  inline Submatrix& ctranspose();
6787 
6788  template< typename Other > inline Submatrix& scale( const Other& scalar );
6790  //**********************************************************************************************
6791 
6792  private:
6793  //**********************************************************************************************
6795  template< typename MT2 >
6796  struct VectorizedAssign {
6797  enum : bool { value = useOptimizedKernels &&
6798  simdEnabled && MT2::simdEnabled &&
6799  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value };
6800  };
6801  //**********************************************************************************************
6802 
6803  //**********************************************************************************************
6805  template< typename MT2 >
6806  struct VectorizedAddAssign {
6807  enum : bool { value = useOptimizedKernels &&
6808  simdEnabled && MT2::simdEnabled &&
6809  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value &&
6810  HasSIMDAdd< ElementType, ElementType_<MT2> >::value &&
6811  !IsDiagonal<MT2>::value };
6812  };
6813  //**********************************************************************************************
6814 
6815  //**********************************************************************************************
6817  template< typename MT2 >
6818  struct VectorizedSubAssign {
6819  enum : bool { value = useOptimizedKernels &&
6820  simdEnabled && MT2::simdEnabled &&
6821  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value &&
6822  HasSIMDSub< ElementType, ElementType_<MT2> >::value &&
6823  !IsDiagonal<MT2>::value };
6824  };
6825  //**********************************************************************************************
6826 
6827  //**********************************************************************************************
6829  template< typename MT2 >
6830  struct VectorizedSchurAssign {
6831  enum : bool { value = useOptimizedKernels &&
6832  simdEnabled && MT2::simdEnabled &&
6833  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value &&
6834  HasSIMDMult< ElementType, ElementType_<MT2> >::value };
6835  };
6836  //**********************************************************************************************
6837 
6838  //**SIMD properties*****************************************************************************
6840  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
6841  //**********************************************************************************************
6842 
6843  public:
6844  //**Expression template evaluation functions****************************************************
6847  template< typename Other >
6848  inline bool canAlias( const Other* alias ) const noexcept;
6849 
6850  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
6851  inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
6852 
6853  template< typename Other >
6854  inline bool isAliased( const Other* alias ) const noexcept;
6855 
6856  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
6857  inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
6858 
6859  inline bool isAligned () const noexcept;
6860  inline bool canSMPAssign() const noexcept;
6861 
6862  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
6863  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
6864  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
6865 
6866  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
6867  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
6868  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
6869  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
6870 
6871  template< typename MT2 >
6872  inline DisableIf_< VectorizedAssign<MT2> > assign( const DenseMatrix<MT2,false>& rhs );
6873 
6874  template< typename MT2 >
6875  inline EnableIf_< VectorizedAssign<MT2> > assign( const DenseMatrix<MT2,false>& rhs );
6876 
6877  template< typename MT2 > inline void assign( const DenseMatrix<MT2,true>& rhs );
6878  template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
6879  template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
6880 
6881  template< typename MT2 >
6882  inline DisableIf_< VectorizedAddAssign<MT2> > addAssign( const DenseMatrix<MT2,false>& rhs );
6883 
6884  template< typename MT2 >
6885  inline EnableIf_< VectorizedAddAssign<MT2> > addAssign( const DenseMatrix<MT2,false>& rhs );
6886 
6887  template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,true>& rhs );
6888  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
6889  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
6890 
6891  template< typename MT2 >
6892  inline DisableIf_< VectorizedSubAssign<MT2> > subAssign( const DenseMatrix<MT2,false>& rhs );
6893 
6894  template< typename MT2 >
6895  inline EnableIf_< VectorizedSubAssign<MT2> > subAssign( const DenseMatrix<MT2,false>& rhs );
6896 
6897  template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,true>& rhs );
6898  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
6899  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
6900 
6901  template< typename MT2 >
6902  inline DisableIf_< VectorizedSchurAssign<MT2> > schurAssign( const DenseMatrix<MT2,false>& rhs );
6903 
6904  template< typename MT2 >
6905  inline EnableIf_< VectorizedSchurAssign<MT2> > schurAssign( const DenseMatrix<MT2,false>& rhs );
6906 
6907  template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,true>& rhs );
6908  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
6909  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
6911  //**********************************************************************************************
6912 
6913  private:
6914  //**Utility functions***************************************************************************
6917  inline bool hasOverlap() const noexcept;
6919  //**********************************************************************************************
6920 
6921  //**Member variables****************************************************************************
6924  Operand matrix_;
6925 
6926  //**********************************************************************************************
6927 
6928  //**Friend declarations*************************************************************************
6929  template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
6930  //**********************************************************************************************
6931 
6932  //**Compile time checks*************************************************************************
6940  //**********************************************************************************************
6941 };
6943 //*************************************************************************************************
6944 
6945 
6946 
6947 
6948 //=================================================================================================
6949 //
6950 // CONSTRUCTORS
6951 //
6952 //=================================================================================================
6953 
6954 //*************************************************************************************************
6967 template< typename MT // Type of the dense matrix
6968  , size_t... CSAs > // Compile time submatrix arguments
6969 template< typename... RSAs > // Runtime submatrix arguments
6970 inline Submatrix<MT,aligned,false,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
6971  : DataType( args... ) // Base class initialization
6972  , matrix_ ( matrix ) // The matrix containing the submatrix
6973 {
6974  if( !Contains< TypeList<RSAs...>, Unchecked >::value ) {
6975  if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
6976  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
6977  }
6978  if( ( simdEnabled && matrix_.data() != nullptr && !checkAlignment( data() ) ) ||
6979  ( rows() > 1UL && matrix_.spacing() % SIMDSIZE != 0UL ) ) {
6980  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix alignment" );
6981  }
6982  }
6983  else {
6984  BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
6985  BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
6986 
6987  BLAZE_USER_ASSERT( !simdEnabled || matrix_.data() == nullptr || checkAlignment( data() ), "Invalid submatrix alignment" );
6988  BLAZE_USER_ASSERT( rows() <= 1UL || matrix_.spacing() % SIMDSIZE == 0UL, "Invalid submatrix alignment" );
6989  }
6990 }
6992 //*************************************************************************************************
6993 
6994 
6995 
6996 
6997 //=================================================================================================
6998 //
6999 // DATA ACCESS FUNCTIONS
7000 //
7001 //=================================================================================================
7002 
7003 //*************************************************************************************************
7014 template< typename MT // Type of the dense matrix
7015  , size_t... CSAs > // Compile time submatrix arguments
7016 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Reference
7017  Submatrix<MT,aligned,false,true,CSAs...>::operator()( size_t i, size_t j )
7018 {
7019  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
7020  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
7021 
7022  return matrix_(row()+i,column()+j);
7023 }
7025 //*************************************************************************************************
7026 
7027 
7028 //*************************************************************************************************
7039 template< typename MT // Type of the dense matrix
7040  , size_t... CSAs > // Compile time submatrix arguments
7041 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstReference
7042  Submatrix<MT,aligned,false,true,CSAs...>::operator()( size_t i, size_t j ) const
7043 {
7044  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
7045  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
7046 
7047  return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
7048 }
7050 //*************************************************************************************************
7051 
7052 
7053 //*************************************************************************************************
7065 template< typename MT // Type of the dense matrix
7066  , size_t... CSAs > // Compile time submatrix arguments
7067 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Reference
7068  Submatrix<MT,aligned,false,true,CSAs...>::at( size_t i, size_t j )
7069 {
7070  if( i >= rows() ) {
7071  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
7072  }
7073  if( j >= columns() ) {
7074  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
7075  }
7076  return (*this)(i,j);
7077 }
7079 //*************************************************************************************************
7080 
7081 
7082 //*************************************************************************************************
7094 template< typename MT // Type of the dense matrix
7095  , size_t... CSAs > // Compile time submatrix arguments
7096 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstReference
7097  Submatrix<MT,aligned,false,true,CSAs...>::at( size_t i, size_t j ) const
7098 {
7099  if( i >= rows() ) {
7100  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
7101  }
7102  if( j >= columns() ) {
7103  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
7104  }
7105  return (*this)(i,j);
7106 }
7108 //*************************************************************************************************
7109 
7110 
7111 //*************************************************************************************************
7121 template< typename MT // Type of the dense matrix
7122  , size_t... CSAs > // Compile time submatrix arguments
7123 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Pointer
7125 {
7126  return matrix_.data() + row()*spacing() + column();
7127 }
7129 //*************************************************************************************************
7130 
7131 
7132 //*************************************************************************************************
7142 template< typename MT // Type of the dense matrix
7143  , size_t... CSAs > // Compile time submatrix arguments
7144 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstPointer
7146 {
7147  return matrix_.data() + row()*spacing() + column();
7148 }
7150 //*************************************************************************************************
7151 
7152 
7153 //*************************************************************************************************
7162 template< typename MT // Type of the dense matrix
7163  , size_t... CSAs > // Compile time submatrix arguments
7164 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Pointer
7166 {
7167  return matrix_.data() + (row()+i)*spacing() + column();
7168 }
7170 //*************************************************************************************************
7171 
7172 
7173 //*************************************************************************************************
7182 template< typename MT // Type of the dense matrix
7183  , size_t... CSAs > // Compile time submatrix arguments
7184 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstPointer
7185  Submatrix<MT,aligned,false,true,CSAs...>::data( size_t i ) const noexcept
7186 {
7187  return matrix_.data() + (row()+i)*spacing() + column();
7188 }
7190 //*************************************************************************************************
7191 
7192 
7193 //*************************************************************************************************
7205 template< typename MT // Type of the dense matrix
7206  , size_t... CSAs > // Compile time submatrix arguments
7207 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Iterator
7209 {
7210  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7211  return ( matrix_.begin( row() + i ) + column() );
7212 }
7214 //*************************************************************************************************
7215 
7216 
7217 //*************************************************************************************************
7229 template< typename MT // Type of the dense matrix
7230  , size_t... CSAs > // Compile time submatrix arguments
7231 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7233 {
7234  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7235  return ( matrix_.cbegin( row() + i ) + column() );
7236 }
7238 //*************************************************************************************************
7239 
7240 
7241 //*************************************************************************************************
7253 template< typename MT // Type of the dense matrix
7254  , size_t... CSAs > // Compile time submatrix arguments
7255 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7257 {
7258  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7259  return ( matrix_.cbegin( row() + i ) + column() );
7260 }
7262 //*************************************************************************************************
7263 
7264 
7265 //*************************************************************************************************
7277 template< typename MT // Type of the dense matrix
7278  , size_t... CSAs > // Compile time submatrix arguments
7279 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Iterator
7281 {
7282  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7283  return ( matrix_.begin( row() + i ) + column() + columns() );
7284 }
7286 //*************************************************************************************************
7287 
7288 
7289 //*************************************************************************************************
7301 template< typename MT // Type of the dense matrix
7302  , size_t... CSAs > // Compile time submatrix arguments
7303 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7305 {
7306  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7307  return ( matrix_.cbegin( row() + i ) + column() + columns() );
7308 }
7310 //*************************************************************************************************
7311 
7312 
7313 //*************************************************************************************************
7325 template< typename MT // Type of the dense matrix
7326  , size_t... CSAs > // Compile time submatrix arguments
7327 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7329 {
7330  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7331  return ( matrix_.cbegin( row() + i ) + column() + columns() );
7332 }
7334 //*************************************************************************************************
7335 
7336 
7337 
7338 
7339 //=================================================================================================
7340 //
7341 // ASSIGNMENT OPERATORS
7342 //
7343 //=================================================================================================
7344 
7345 //*************************************************************************************************
7356 template< typename MT // Type of the dense matrix
7357  , size_t... CSAs > // Compile time submatrix arguments
7358 inline Submatrix<MT,aligned,false,true,CSAs...>&
7359  Submatrix<MT,aligned,false,true,CSAs...>::operator=( const ElementType& rhs )
7360 {
7361  const size_t iend( row() + rows() );
7362  decltype(auto) left( derestrict( matrix_ ) );
7363 
7364  for( size_t i=row(); i<iend; ++i )
7365  {
7366  const size_t jbegin( ( IsUpper<MT>::value )
7367  ?( ( IsUniUpper<MT>::value || IsStrictlyUpper<MT>::value )
7368  ?( max( i+1UL, column() ) )
7369  :( max( i, column() ) ) )
7370  :( column() ) );
7371  const size_t jend ( ( IsLower<MT>::value )
7372  ?( ( IsUniLower<MT>::value || IsStrictlyLower<MT>::value )
7373  ?( min( i, column()+columns() ) )
7374  :( min( i+1UL, column()+columns() ) ) )
7375  :( column()+columns() ) );
7376 
7377  for( size_t j=jbegin; j<jend; ++j ) {
7378  if( !IsRestricted<MT>::value || IsTriangular<MT>::value || trySet( matrix_, i, j, rhs ) )
7379  left(i,j) = rhs;
7380  }
7381  }
7382 
7383  return *this;
7384 }
7386 //*************************************************************************************************
7387 
7388 
7389 //*************************************************************************************************
7405 template< typename MT // Type of the dense matrix
7406  , size_t... CSAs > // Compile time submatrix arguments
7407 inline Submatrix<MT,aligned,false,true,CSAs...>&
7408  Submatrix<MT,aligned,false,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
7409 {
7410  if( list.size() != rows() ) {
7411  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
7412  }
7413 
7414  if( IsRestricted<MT>::value ) {
7415  const InitializerMatrix<ElementType> tmp( list, columns() );
7416  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7417  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7418  }
7419  }
7420 
7421  decltype(auto) left( derestrict( *this ) );
7422  size_t i( 0UL );
7423 
7424  for( const auto& rowList : list ) {
7425  std::fill( std::copy( rowList.begin(), rowList.end(), left.begin(i) ), left.end(i), ElementType() );
7426  ++i;
7427  }
7428 
7429  return *this;
7430 }
7432 //*************************************************************************************************
7433 
7434 
7435 //*************************************************************************************************
7450 template< typename MT // Type of the dense matrix
7451  , size_t... CSAs > // Compile time submatrix arguments
7452 inline Submatrix<MT,aligned,false,true,CSAs...>&
7453  Submatrix<MT,aligned,false,true,CSAs...>::operator=( const Submatrix& rhs )
7454 {
7457 
7458  if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
7459  return *this;
7460 
7461  if( rows() != rhs.rows() || columns() != rhs.columns() ) {
7462  BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
7463  }
7464 
7465  if( !tryAssign( matrix_, rhs, row(), column() ) ) {
7466  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7467  }
7468 
7469  decltype(auto) left( derestrict( *this ) );
7470 
7471  if( rhs.canAlias( &matrix_ ) ) {
7472  const ResultType tmp( rhs );
7473  smpAssign( left, tmp );
7474  }
7475  else {
7476  smpAssign( left, rhs );
7477  }
7478 
7479  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7480 
7481  return *this;
7482 }
7484 //*************************************************************************************************
7485 
7486 
7487 //*************************************************************************************************
7502 template< typename MT // Type of the dense matrix
7503  , size_t... CSAs > // Compile time submatrix arguments
7504 template< typename MT2 // Type of the right-hand side matrix
7505  , bool SO > // Storage order of the right-hand side matrix
7506 inline Submatrix<MT,aligned,false,true,CSAs...>&
7507  Submatrix<MT,aligned,false,true,CSAs...>::operator=( const Matrix<MT2,SO>& rhs )
7508 {
7510 
7511  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7512  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7513  }
7514 
7515  using Right = If_< IsRestricted<MT>, CompositeType_<MT2>, const MT2& >;
7516  Right right( ~rhs );
7517 
7518  if( !tryAssign( matrix_, right, row(), column() ) ) {
7519  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7520  }
7521 
7522  decltype(auto) left( derestrict( *this ) );
7523 
7524  if( IsReference<Right>::value && right.canAlias( &matrix_ ) ) {
7525  const ResultType_<MT2> tmp( right );
7526  if( IsSparseMatrix<MT2>::value )
7527  reset();
7528  smpAssign( left, tmp );
7529  }
7530  else {
7531  if( IsSparseMatrix<MT2>::value )
7532  reset();
7533  smpAssign( left, right );
7534  }
7535 
7536  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7537 
7538  return *this;
7539 }
7541 //*************************************************************************************************
7542 
7543 
7544 //*************************************************************************************************
7558 template< typename MT // Type of the dense matrix
7559  , size_t... CSAs > // Compile time submatrix arguments
7560 template< typename MT2 // Type of the right-hand side matrix
7561  , bool SO > // Storage order of the right-hand side matrix
7562 inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,aligned,false,true,CSAs...>& >
7563  Submatrix<MT,aligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
7564 {
7568 
7569  using AddType = AddTrait_< ResultType, ResultType_<MT2> >;
7570 
7573 
7574  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7575  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7576  }
7577 
7578  if( !tryAddAssign( matrix_, ~rhs, row(), column() ) ) {
7579  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7580  }
7581 
7582  decltype(auto) left( derestrict( *this ) );
7583 
7584  if( ( ( IsSymmetric<MT>::value || IsHermitian<MT>::value ) && hasOverlap() ) ||
7585  (~rhs).canAlias( &matrix_ ) ) {
7586  const AddType tmp( *this + (~rhs) );
7587  smpAssign( left, tmp );
7588  }
7589  else {
7590  smpAddAssign( left, ~rhs );
7591  }
7592 
7593  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7594 
7595  return *this;
7596 }
7598 //*************************************************************************************************
7599 
7600 
7601 //*************************************************************************************************
7615 template< typename MT // Type of the dense matrix
7616  , size_t... CSAs > // Compile time submatrix arguments
7617 template< typename MT2 // Type of the right-hand side matrix
7618  , bool SO > // Storage order of the right-hand side matrix
7619 inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,aligned,false,true,CSAs...>& >
7620  Submatrix<MT,aligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
7621 {
7625 
7626  using AddType = AddTrait_< ResultType, ResultType_<MT2> >;
7627 
7630 
7631  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7632  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7633  }
7634 
7635  const AddType tmp( *this + (~rhs) );
7636 
7637  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7638  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7639  }
7640 
7641  decltype(auto) left( derestrict( *this ) );
7642 
7643  smpAssign( left, tmp );
7644 
7645  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7646 
7647  return *this;
7648 }
7650 //*************************************************************************************************
7651 
7652 
7653 //*************************************************************************************************
7667 template< typename MT // Type of the dense matrix
7668  , size_t... CSAs > // Compile time submatrix arguments
7669 template< typename MT2 // Type of the right-hand side matrix
7670  , bool SO > // Storage order of the right-hand side matrix
7671 inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,aligned,false,true,CSAs...>& >
7672  Submatrix<MT,aligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
7673 {
7677 
7678  using SubType = SubTrait_< ResultType, ResultType_<MT2> >;
7679 
7682 
7683  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7684  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7685  }
7686 
7687  if( !trySubAssign( matrix_, ~rhs, row(), column() ) ) {
7688  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7689  }
7690 
7691  decltype(auto) left( derestrict( *this ) );
7692 
7693  if( ( ( IsSymmetric<MT>::value || IsHermitian<MT>::value ) && hasOverlap() ) ||
7694  (~rhs).canAlias( &matrix_ ) ) {
7695  const SubType tmp( *this - (~rhs ) );
7696  smpAssign( left, tmp );
7697  }
7698  else {
7699  smpSubAssign( left, ~rhs );
7700  }
7701 
7702  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7703 
7704  return *this;
7705 }
7707 //*************************************************************************************************
7708 
7709 
7710 //*************************************************************************************************
7724 template< typename MT // Type of the dense matrix
7725  , size_t... CSAs > // Compile time submatrix arguments
7726 template< typename MT2 // Type of the right-hand side matrix
7727  , bool SO > // Storage order of the right-hand side matrix
7728 inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,aligned,false,true,CSAs...>& >
7729  Submatrix<MT,aligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
7730 {
7734 
7735  using SubType = SubTrait_< ResultType, ResultType_<MT2> >;
7736 
7739 
7740  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7741  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7742  }
7743 
7744  const SubType tmp( *this - (~rhs) );
7745 
7746  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7747  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7748  }
7749 
7750  decltype(auto) left( derestrict( *this ) );
7751 
7752  smpAssign( left, tmp );
7753 
7754  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7755 
7756  return *this;
7757 }
7759 //*************************************************************************************************
7760 
7761 
7762 //*************************************************************************************************
7776 template< typename MT // Type of the dense matrix
7777  , size_t... CSAs > // Compile time submatrix arguments
7778 template< typename MT2 // Type of the right-hand side matrix
7779  , bool SO > // Storage order of the right-hand side matrix
7780 inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,aligned,false,true,CSAs...>& >
7781  Submatrix<MT,aligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
7782 {
7786 
7787  using SchurType = SchurTrait_< ResultType, ResultType_<MT2> >;
7788 
7790 
7791  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7792  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7793  }
7794 
7795  if( !trySchurAssign( matrix_, ~rhs, row(), column() ) ) {
7796  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7797  }
7798 
7799  decltype(auto) left( derestrict( *this ) );
7800 
7801  if( ( ( IsSymmetric<MT>::value || IsHermitian<MT>::value ) && hasOverlap() ) ||
7802  (~rhs).canAlias( &matrix_ ) ) {
7803  const SchurType tmp( *this % (~rhs) );
7804  if( IsSparseMatrix<SchurType>::value )
7805  reset();
7806  smpAssign( left, tmp );
7807  }
7808  else {
7809  smpSchurAssign( left, ~rhs );
7810  }
7811 
7812  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7813 
7814  return *this;
7815 }
7817 //*************************************************************************************************
7818 
7819 
7820 //*************************************************************************************************
7834 template< typename MT // Type of the dense matrix
7835  , size_t... CSAs > // Compile time submatrix arguments
7836 template< typename MT2 // Type of the right-hand side matrix
7837  , bool SO > // Storage order of the right-hand side matrix
7838 inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,aligned,false,true,CSAs...>& >
7839  Submatrix<MT,aligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
7840 {
7844 
7845  using SchurType = SchurTrait_< ResultType, ResultType_<MT2> >;
7846 
7848 
7849  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7850  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7851  }
7852 
7853  const SchurType tmp( *this % (~rhs) );
7854 
7855  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7856  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7857  }
7858 
7859  decltype(auto) left( derestrict( *this ) );
7860 
7861  if( IsSparseMatrix<SchurType>::value ) {
7862  reset();
7863  }
7864 
7865  smpAssign( left, tmp );
7866 
7867  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7868 
7869  return *this;
7870 }
7872 //*************************************************************************************************
7873 
7874 
7875 
7876 
7877 //=================================================================================================
7878 //
7879 // UTILITY FUNCTIONS
7880 //
7881 //=================================================================================================
7882 
7883 //*************************************************************************************************
7889 template< typename MT // Type of the dense matrix
7890  , size_t... CSAs > // Compile time submatrix arguments
7891 inline MT& Submatrix<MT,aligned,false,true,CSAs...>::operand() noexcept
7892 {
7893  return matrix_;
7894 }
7896 //*************************************************************************************************
7897 
7898 
7899 //*************************************************************************************************
7905 template< typename MT // Type of the dense matrix
7906  , size_t... CSAs > // Compile time submatrix arguments
7907 inline const MT& Submatrix<MT,aligned,false,true,CSAs...>::operand() const noexcept
7908 {
7909  return matrix_;
7910 }
7912 //*************************************************************************************************
7913 
7914 
7915 //*************************************************************************************************
7926 template< typename MT // Type of the dense matrix
7927  , size_t... CSAs > // Compile time submatrix arguments
7928 inline size_t Submatrix<MT,aligned,false,true,CSAs...>::spacing() const noexcept
7929 {
7930  return matrix_.spacing();
7931 }
7933 //*************************************************************************************************
7934 
7935 
7936 //*************************************************************************************************
7942 template< typename MT // Type of the dense matrix
7943  , size_t... CSAs > // Compile time submatrix arguments
7944 inline size_t Submatrix<MT,aligned,false,true,CSAs...>::capacity() const noexcept
7945 {
7946  return rows() * columns();
7947 }
7949 //*************************************************************************************************
7950 
7951 
7952 //*************************************************************************************************
7964 template< typename MT // Type of the dense matrix
7965  , size_t... CSAs > // Compile time submatrix arguments
7966 inline size_t Submatrix<MT,aligned,false,true,CSAs...>::capacity( size_t i ) const noexcept
7967 {
7968  UNUSED_PARAMETER( i );
7969 
7970  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
7971 
7972  return columns();
7973 }
7975 //*************************************************************************************************
7976 
7977 
7978 //*************************************************************************************************
7984 template< typename MT // Type of the dense matrix
7985  , size_t... CSAs > // Compile time submatrix arguments
7987 {
7988  const size_t iend( row() + rows() );
7989  const size_t jend( column() + columns() );
7990  size_t nonzeros( 0UL );
7991 
7992  for( size_t i=row(); i<iend; ++i )
7993  for( size_t j=column(); j<jend; ++j )
7994  if( !isDefault( matrix_(i,j) ) )
7995  ++nonzeros;
7996 
7997  return nonzeros;
7998 }
8000 //*************************************************************************************************
8001 
8002 
8003 //*************************************************************************************************
8015 template< typename MT // Type of the dense matrix
8016  , size_t... CSAs > // Compile time submatrix arguments
8017 inline size_t Submatrix<MT,aligned,false,true,CSAs...>::nonZeros( size_t i ) const
8018 {
8019  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
8020 
8021  const size_t jend( column() + columns() );
8022  size_t nonzeros( 0UL );
8023 
8024  for( size_t j=column(); j<jend; ++j )
8025  if( !isDefault( matrix_(row()+i,j) ) )
8026  ++nonzeros;
8027 
8028  return nonzeros;
8029 }
8031 //*************************************************************************************************
8032 
8033 
8034 //*************************************************************************************************
8040 template< typename MT // Type of the dense matrix
8041  , size_t... CSAs > // Compile time submatrix arguments
8043 {
8044  using blaze::clear;
8045 
8046  for( size_t i=row(); i<row()+rows(); ++i )
8047  {
8048  const size_t jbegin( ( IsUpper<MT>::value )
8049  ?( ( IsUniUpper<MT>::value || IsStrictlyUpper<MT>::value )
8050  ?( max( i+1UL, column() ) )
8051  :( max( i, column() ) ) )
8052  :( column() ) );
8053  const size_t jend ( ( IsLower<MT>::value )
8054  ?( ( IsUniLower<MT>::value || IsStrictlyLower<MT>::value )
8055  ?( min( i, column()+columns() ) )
8056  :( min( i+1UL, column()+columns() ) ) )
8057  :( column()+columns() ) );
8058 
8059  for( size_t j=jbegin; j<jend; ++j )
8060  clear( matrix_(i,j) );
8061  }
8062 }
8064 //*************************************************************************************************
8065 
8066 
8067 //*************************************************************************************************
8079 template< typename MT // Type of the dense matrix
8080  , size_t... CSAs > // Compile time submatrix arguments
8082 {
8083  using blaze::clear;
8084 
8085  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
8086 
8087  const size_t jbegin( ( IsUpper<MT>::value )
8088  ?( ( IsUniUpper<MT>::value || IsStrictlyUpper<MT>::value )
8089  ?( max( i+1UL, column() ) )
8090  :( max( i, column() ) ) )
8091  :( column() ) );
8092  const size_t jend ( ( IsLower<MT>::value )
8093  ?( ( IsUniLower<MT>::value || IsStrictlyLower<MT>::value )
8094  ?( min( i, column()+columns() ) )
8095  :( min( i+1UL, column()+columns() ) ) )
8096  :( column()+columns() ) );
8097 
8098  for( size_t j=jbegin; j<jend; ++j )
8099  clear( matrix_(row()+i,j) );
8100 }
8102 //*************************************************************************************************
8103 
8104 
8105 //*************************************************************************************************
8115 template< typename MT // Type of the dense matrix
8116  , size_t... CSAs > // Compile time submatrix arguments
8117 inline bool Submatrix<MT,aligned,false,true,CSAs...>::hasOverlap() const noexcept
8118 {
8119  BLAZE_INTERNAL_ASSERT( IsSymmetric<MT>::value || IsHermitian<MT>::value, "Invalid matrix detected" );
8120 
8121  if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
8122  return false;
8123  else return true;
8124 }
8126 //*************************************************************************************************
8127 
8128 
8129 
8130 
8131 //=================================================================================================
8132 //
8133 // NUMERIC FUNCTIONS
8134 //
8135 //=================================================================================================
8136 
8137 //*************************************************************************************************
8155 template< typename MT // Type of the dense matrix
8156  , size_t... CSAs > // Compile time submatrix arguments
8157 inline Submatrix<MT,aligned,false,true,CSAs...>&
8159 {
8160  if( rows() != columns() ) {
8161  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
8162  }
8163 
8164  if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
8165  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
8166  }
8167 
8168  decltype(auto) left( derestrict( *this ) );
8169  const ResultType tmp( trans( *this ) );
8170 
8171  smpAssign( left, tmp );
8172 
8173  return *this;
8174 }
8176 //*************************************************************************************************
8177 
8178 
8179 //*************************************************************************************************
8197 template< typename MT // Type of the dense matrix
8198  , size_t... CSAs > // Compile time submatrix arguments
8199 inline Submatrix<MT,aligned,false,true,CSAs...>&
8200  Submatrix<MT,aligned,false,true,CSAs...>::ctranspose()
8201 {
8202  if( rows() != columns() ) {
8203  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
8204  }
8205 
8206  if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
8207  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
8208  }
8209 
8210  decltype(auto) left( derestrict( *this ) );
8211  const ResultType tmp( ctrans( *this ) );
8212 
8213  smpAssign( left, tmp );
8214 
8215  return *this;
8216 }
8218 //*************************************************************************************************
8219 
8220 
8221 //*************************************************************************************************
8234 template< typename MT // Type of the dense matrix
8235  , size_t... CSAs > // Compile time submatrix arguments
8236 template< typename Other > // Data type of the scalar value
8237 inline Submatrix<MT,aligned,false,true,CSAs...>&
8238  Submatrix<MT,aligned,false,true,CSAs...>::scale( const Other& scalar )
8239 {
8241 
8242  const size_t iend( row() + rows() );
8243 
8244  for( size_t i=row(); i<iend; ++i )
8245  {
8246  const size_t jbegin( ( IsUpper<MT>::value )
8247  ?( ( IsStrictlyUpper<MT>::value )
8248  ?( max( i+1UL, column() ) )
8249  :( max( i, column() ) ) )
8250  :( column() ) );
8251  const size_t jend ( ( IsLower<MT>::value )
8252  ?( ( IsStrictlyLower<MT>::value )
8253  ?( min( i, column()+columns() ) )
8254  :( min( i+1UL, column()+columns() ) ) )
8255  :( column()+columns() ) );
8256 
8257  for( size_t j=jbegin; j<jend; ++j )
8258  matrix_(i,j) *= scalar;
8259  }
8260 
8261  return *this;
8262 }
8264 //*************************************************************************************************
8265 
8266 
8267 
8268 
8269 //=================================================================================================
8270 //
8271 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
8272 //
8273 //=================================================================================================
8274 
8275 //*************************************************************************************************
8286 template< typename MT // Type of the dense matrix
8287  , size_t... CSAs > // Compile time submatrix arguments
8288 template< typename Other > // Data type of the foreign expression
8289 inline bool Submatrix<MT,aligned,false,true,CSAs...>::canAlias( const Other* alias ) const noexcept
8290 {
8291  return matrix_.isAliased( alias );
8292 }
8294 //*************************************************************************************************
8295 
8296 
8297 //*************************************************************************************************
8308 template< typename MT // Type of the dense matrix
8309  , size_t... CSAs > // Compile time submatrix arguments
8310 template< typename MT2 // Data type of the foreign dense submatrix
8311  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
8312  , bool SO2 // Storage order of the foreign dense submatrix
8313  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
8314 inline bool
8315  Submatrix<MT,aligned,false,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
8316 {
8317  return ( matrix_.isAliased( &alias->matrix_ ) &&
8318  ( row() + rows() > alias->row() ) &&
8319  ( row() < alias->row() + alias->rows() ) &&
8320  ( column() + columns() > alias->column() ) &&
8321  ( column() < alias->column() + alias->columns() ) );
8322 }
8324 //*************************************************************************************************
8325 
8326 
8327 //*************************************************************************************************
8338 template< typename MT // Type of the dense matrix
8339  , size_t... CSAs > // Compile time submatrix arguments
8340 template< typename Other > // Data type of the foreign expression
8341 inline bool Submatrix<MT,aligned,false,true,CSAs...>::isAliased( const Other* alias ) const noexcept
8342 {
8343  return matrix_.isAliased( alias );
8344 }
8346 //*************************************************************************************************
8347 
8348 
8349 //*************************************************************************************************
8360 template< typename MT // Type of the dense matrix
8361  , size_t... CSAs > // Compile time submatrix arguments
8362 template< typename MT2 // Data type of the foreign dense submatrix
8363  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
8364  , bool SO2 // Storage order of the foreign dense submatrix
8365  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
8366 inline bool
8367  Submatrix<MT,aligned,false,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
8368 {
8369  return ( matrix_.isAliased( &alias->matrix_ ) &&
8370  ( row() + rows() > alias->row() ) &&
8371  ( row() < alias->row() + alias->rows() ) &&
8372  ( column() + columns() > alias->column() ) &&
8373  ( column() < alias->column() + alias->columns() ) );
8374 }
8376 //*************************************************************************************************
8377 
8378 
8379 //*************************************************************************************************
8389 template< typename MT // Type of the dense matrix
8390  , size_t... CSAs > // Compile time submatrix arguments
8391 inline bool Submatrix<MT,aligned,false,true,CSAs...>::isAligned() const noexcept
8392 {
8393  return true;
8394 }
8396 //*************************************************************************************************
8397 
8398 
8399 //*************************************************************************************************
8410 template< typename MT // Type of the dense matrix
8411  , size_t... CSAs > // Compile time submatrix arguments
8412 inline bool Submatrix<MT,aligned,false,true,CSAs...>::canSMPAssign() const noexcept
8413 {
8414  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
8415 }
8417 //*************************************************************************************************
8418 
8419 
8420 //*************************************************************************************************
8436 template< typename MT // Type of the dense matrix
8437  , size_t... CSAs > // Compile time submatrix arguments
8438 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,false,true,CSAs...>::SIMDType
8439  Submatrix<MT,aligned,false,true,CSAs...>::load( size_t i, size_t j ) const noexcept
8440 {
8441  return loada( i, j );
8442 }
8444 //*************************************************************************************************
8445 
8446 
8447 //*************************************************************************************************
8463 template< typename MT // Type of the dense matrix
8464  , size_t... CSAs > // Compile time submatrix arguments
8465 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,false,true,CSAs...>::SIMDType
8466  Submatrix<MT,aligned,false,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
8467 {
8469 
8470  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8471  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8472  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8473  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8474 
8475  return matrix_.loada( row()+i, column()+j );
8476 }
8478 //*************************************************************************************************
8479 
8480 
8481 //*************************************************************************************************
8497 template< typename MT // Type of the dense matrix
8498  , size_t... CSAs > // Compile time submatrix arguments
8499 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,false,true,CSAs...>::SIMDType
8500  Submatrix<MT,aligned,false,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
8501 {
8503 
8504  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8505  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8506  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8507  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8508 
8509  return matrix_.loadu( row()+i, column()+j );
8510 }
8512 //*************************************************************************************************
8513 
8514 
8515 //*************************************************************************************************
8532 template< typename MT // Type of the dense matrix
8533  , size_t... CSAs > // Compile time submatrix arguments
8535  Submatrix<MT,aligned,false,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
8536 {
8537  return storea( i, j, value );
8538 }
8540 //*************************************************************************************************
8541 
8542 
8543 //*************************************************************************************************
8560 template< typename MT // Type of the dense matrix
8561  , size_t... CSAs > // Compile time submatrix arguments
8563  Submatrix<MT,aligned,false,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
8564 {
8566 
8567  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8568  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8569  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8570  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8571 
8572  return matrix_.storea( row()+i, column()+j, value );
8573 }
8575 //*************************************************************************************************
8576 
8577 
8578 //*************************************************************************************************
8595 template< typename MT // Type of the dense matrix
8596  , size_t... CSAs > // Compile time submatrix arguments
8598  Submatrix<MT,aligned,false,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
8599 {
8601 
8602  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8603  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8604  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8605  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8606 
8607  matrix_.storeu( row()+i, column()+j, value );
8608 }
8610 //*************************************************************************************************
8611 
8612 
8613 //*************************************************************************************************
8631 template< typename MT // Type of the dense matrix
8632  , size_t... CSAs > // Compile time submatrix arguments
8634  Submatrix<MT,aligned,false,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
8635 {
8637 
8638  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8639  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8640  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8641  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8642 
8643  matrix_.stream( row()+i, column()+j, value );
8644 }
8646 //*************************************************************************************************
8647 
8648 
8649 //*************************************************************************************************
8661 template< typename MT // Type of the dense matrix
8662  , size_t... CSAs > // Compile time submatrix arguments
8663 template< typename MT2 > // Type of the right-hand side dense matrix
8664 inline DisableIf_< typename Submatrix<MT,aligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedAssign<MT2> >
8665  Submatrix<MT,aligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
8666 {
8667  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8668  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8669 
8670  const size_t jpos( columns() & size_t(-2) );
8671  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
8672 
8673  for( size_t i=0UL; i<rows(); ++i ) {
8674  for( size_t j=0UL; j<jpos; j+=2UL ) {
8675  matrix_(row()+i,column()+j ) = (~rhs)(i,j );
8676  matrix_(row()+i,column()+j+1UL) = (~rhs)(i,j+1UL);
8677  }
8678  if( jpos < columns() ) {
8679  matrix_(row()+i,column()+jpos) = (~rhs)(i,jpos);
8680  }
8681  }
8682 }
8684 //*************************************************************************************************
8685 
8686 
8687 //*************************************************************************************************
8699 template< typename MT // Type of the dense matrix
8700  , size_t... CSAs > // Compile time submatrix arguments
8701 template< typename MT2 > // Type of the right-hand side dense matrix
8702 inline EnableIf_< typename Submatrix<MT,aligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedAssign<MT2> >
8703  Submatrix<MT,aligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
8704 {
8706 
8707  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8708  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8709 
8710  const size_t jpos( columns() & size_t(-SIMDSIZE) );
8711  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
8712 
8713  if( useStreaming &&
8714  rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
8715  !(~rhs).isAliased( &matrix_ ) )
8716  {
8717  for( size_t i=0UL; i<rows(); ++i )
8718  {
8719  size_t j( 0UL );
8720  Iterator left( begin(i) );
8721  ConstIterator_<MT2> right( (~rhs).begin(i) );
8722 
8723  for( ; j<jpos; j+=SIMDSIZE ) {
8724  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8725  }
8726  for( ; j<columns(); ++j ) {
8727  *left = *right; ++left; ++right;
8728  }
8729  }
8730  }
8731  else
8732  {
8733  for( size_t i=0UL; i<rows(); ++i )
8734  {
8735  size_t j( 0UL );
8736  Iterator left( begin(i) );
8737  ConstIterator_<MT2> right( (~rhs).begin(i) );
8738 
8739  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
8740  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8741  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8742  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8743  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8744  }
8745  for( ; j<jpos; j+=SIMDSIZE ) {
8746  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8747  }
8748  for( ; j<columns(); ++j ) {
8749  *left = *right; ++left; ++right;
8750  }
8751  }
8752  }
8753 }
8755 //*************************************************************************************************
8756 
8757 
8758 //*************************************************************************************************
8770 template< typename MT // Type of the dense matrix
8771  , size_t... CSAs > // Compile time submatrix arguments
8772 template< typename MT2 > // Type of the right-hand side dense matrix
8773 inline void Submatrix<MT,aligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
8774 {
8776 
8777  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8778  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8779 
8780  constexpr size_t block( BLOCK_SIZE );
8781 
8782  for( size_t ii=0UL; ii<rows(); ii+=block ) {
8783  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
8784  for( size_t jj=0UL; jj<columns(); jj+=block ) {
8785  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
8786  for( size_t i=ii; i<iend; ++i ) {
8787  for( size_t j=jj; j<jend; ++j ) {
8788  matrix_(row()+i,column()+j) = (~rhs)(i,j);
8789  }
8790  }
8791  }
8792  }
8793 }
8795 //*************************************************************************************************
8796 
8797 
8798 //*************************************************************************************************
8810 template< typename MT // Type of the dense matrix
8811  , size_t... CSAs > // Compile time submatrix arguments
8812 template< typename MT2 > // Type of the right-hand side sparse matrix
8813 inline void Submatrix<MT,aligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
8814 {
8815  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8816  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8817 
8818  for( size_t i=0UL; i<rows(); ++i )
8819  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
8820  matrix_(row()+i,column()+element->index()) = element->value();
8821 }
8823 //*************************************************************************************************
8824 
8825 
8826 //*************************************************************************************************
8838 template< typename MT // Type of the dense matrix
8839  , size_t... CSAs > // Compile time submatrix arguments
8840 template< typename MT2 > // Type of the right-hand side sparse matrix
8841 inline void Submatrix<MT,aligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
8842 {
8844 
8845  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8846  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8847 
8848  for( size_t j=0UL; j<columns(); ++j )
8849  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
8850  matrix_(row()+element->index(),column()+j) = element->value();
8851 }
8853 //*************************************************************************************************
8854 
8855 
8856 //*************************************************************************************************
8868 template< typename MT // Type of the dense matrix
8869  , size_t... CSAs > // Compile time submatrix arguments
8870 template< typename MT2 > // Type of the right-hand side dense matrix
8871 inline DisableIf_< typename Submatrix<MT,aligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedAddAssign<MT2> >
8872  Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
8873 {
8874  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8875  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8876 
8877  const size_t jpos( columns() & size_t(-2) );
8878  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
8879 
8880  for( size_t i=0UL; i<rows(); ++i )
8881  {
8882  if( IsDiagonal<MT2>::value ) {
8883  matrix_(row()+i,column()+i) += (~rhs)(i,i);
8884  }
8885  else {
8886  for( size_t j=0UL; j<jpos; j+=2UL ) {
8887  matrix_(row()+i,column()+j ) += (~rhs)(i,j );
8888  matrix_(row()+i,column()+j+1UL) += (~rhs)(i,j+1UL);
8889  }
8890  if( jpos < columns() ) {
8891  matrix_(row()+i,column()+jpos) += (~rhs)(i,jpos);
8892  }
8893  }
8894  }
8895 }
8897 //*************************************************************************************************
8898 
8899 
8900 //*************************************************************************************************
8912 template< typename MT // Type of the dense matrix
8913  , size_t... CSAs > // Compile time submatrix arguments
8914 template< typename MT2 > // Type of the right-hand side dense matrix
8915 inline EnableIf_< typename Submatrix<MT,aligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedAddAssign<MT2> >
8916  Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
8917 {
8919 
8920  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8921  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8922 
8923  for( size_t i=0UL; i<rows(); ++i )
8924  {
8925  const size_t jbegin( ( IsUpper<MT2>::value )
8926  ?( ( IsStrictlyUpper<MT2>::value ? i+1UL : i ) & size_t(-SIMDSIZE) )
8927  :( 0UL ) );
8928  const size_t jend ( ( IsLower<MT2>::value )
8929  ?( IsStrictlyLower<MT2>::value ? i : i+1UL )
8930  :( columns() ) );
8931  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
8932 
8933  const size_t jpos( jend & size_t(-SIMDSIZE) );
8934  BLAZE_INTERNAL_ASSERT( ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
8935 
8936  size_t j( jbegin );
8937  Iterator left( begin(i) + jbegin );
8938  ConstIterator_<MT2> right( (~rhs).begin(i) + jbegin );
8939 
8940  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
8941  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8942  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8943  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8944  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8945  }
8946  for( ; j<jpos; j+=SIMDSIZE ) {
8947  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8948  }
8949  for( ; j<jend; ++j ) {
8950  *left += *right; ++left; ++right;
8951  }
8952  }
8953 }
8955 //*************************************************************************************************
8956 
8957 
8958 //*************************************************************************************************
8970 template< typename MT // Type of the dense matrix
8971  , size_t... CSAs > // Compile time submatrix arguments
8972 template< typename MT2 > // Type of the right-hand side dense matrix
8973 inline void Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
8974 {
8976 
8977  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8978  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8979 
8980  constexpr size_t block( BLOCK_SIZE );
8981 
8982  for( size_t ii=0UL; ii<rows(); ii+=block ) {
8983  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
8984  for( size_t jj=0UL; jj<columns(); jj+=block ) {
8985  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
8986  for( size_t i=ii; i<iend; ++i ) {
8987  for( size_t j=jj; j<jend; ++j ) {
8988  matrix_(row()+i,column()+j) += (~rhs)(i,j);
8989  }
8990  }
8991  }
8992  }
8993 }
8995 //*************************************************************************************************
8996 
8997 
8998 //*************************************************************************************************
9010 template< typename MT // Type of the dense matrix
9011  , size_t... CSAs > // Compile time submatrix arguments
9012 template< typename MT2 > // Type of the right-hand side sparse matrix
9013 inline void Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
9014 {
9015  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9016  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9017 
9018  for( size_t i=0UL; i<rows(); ++i )
9019  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
9020  matrix_(row()+i,column()+element->index()) += element->value();
9021 }
9023 //*************************************************************************************************
9024 
9025 
9026 //*************************************************************************************************
9038 template< typename MT // Type of the dense matrix
9039  , size_t... CSAs > // Compile time submatrix arguments
9040 template< typename MT2 > // Type of the right-hand side sparse matrix
9041 inline void Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
9042 {
9044 
9045  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9046  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9047 
9048  for( size_t j=0UL; j<columns(); ++j )
9049  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
9050  matrix_(row()+element->index(),column()+j) += element->value();
9051 }
9053 //*************************************************************************************************
9054 
9055 
9056 //*************************************************************************************************
9068 template< typename MT // Type of the dense matrix
9069  , size_t... CSAs > // Compile time submatrix arguments
9070 template< typename MT2 > // Type of the right-hand side dense matrix
9071 inline DisableIf_< typename Submatrix<MT,aligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedSubAssign<MT2> >
9072  Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
9073 {
9074  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9075  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9076 
9077  const size_t jpos( columns() & size_t(-2) );
9078  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
9079 
9080  for( size_t i=0UL; i<rows(); ++i )
9081  {
9082  if( IsDiagonal<MT2>::value ) {
9083  matrix_(row()+i,column()+i) -= (~rhs)(i,i);
9084  }
9085  else {
9086  for( size_t j=0UL; j<jpos; j+=2UL ) {
9087  matrix_(row()+i,column()+j ) -= (~rhs)(i,j );
9088  matrix_(row()+i,column()+j+1UL) -= (~rhs)(i,j+1UL);
9089  }
9090  if( jpos < columns() ) {
9091  matrix_(row()+i,column()+jpos) -= (~rhs)(i,jpos);
9092  }
9093  }
9094  }
9095 }
9097 //*************************************************************************************************
9098 
9099 
9100 //*************************************************************************************************
9112 template< typename MT // Type of the dense matrix
9113  , size_t... CSAs > // Compile time submatrix arguments
9114 template< typename MT2 > // Type of the right-hand side dense matrix
9115 inline EnableIf_< typename Submatrix<MT,aligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedSubAssign<MT2> >
9116  Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
9117 {
9119 
9120  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9121  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9122 
9123  for( size_t i=0UL; i<rows(); ++i )
9124  {
9125  const size_t jbegin( ( IsUpper<MT2>::value )
9126  ?( ( IsStrictlyUpper<MT2>::value ? i+1UL : i ) & size_t(-SIMDSIZE) )
9127  :( 0UL ) );
9128  const size_t jend ( ( IsLower<MT2>::value )
9129  ?( IsStrictlyLower<MT2>::value ? i : i+1UL )
9130  :( columns() ) );
9131  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
9132 
9133  const size_t jpos( jend & size_t(-SIMDSIZE) );
9134  BLAZE_INTERNAL_ASSERT( ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
9135 
9136  size_t j( jbegin );
9137  Iterator left( begin(i) + jbegin );
9138  ConstIterator_<MT2> right( (~rhs).begin(i) + jbegin );
9139 
9140  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
9141  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9142  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9143  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9144  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9145  }
9146  for( ; j<jpos; j+=SIMDSIZE ) {
9147  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9148  }
9149  for( ; j<jend; ++j ) {
9150  *left -= *right; ++left; ++right;
9151  }
9152  }
9153 }
9155 //*************************************************************************************************
9156 
9157 
9158 //*************************************************************************************************
9170 template< typename MT // Type of the dense matrix
9171  , size_t... CSAs > // Compile time submatrix arguments
9172 template< typename MT2 > // Type of the right-hand side dense matrix
9173 inline void Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
9174 {
9176 
9177  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9178  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9179 
9180  constexpr size_t block( BLOCK_SIZE );
9181 
9182  for( size_t ii=0UL; ii<rows(); ii+=block ) {
9183  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
9184  for( size_t jj=0UL; jj<columns(); jj+=block ) {
9185  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
9186  for( size_t i=ii; i<iend; ++i ) {
9187  for( size_t j=jj; j<jend; ++j ) {
9188  matrix_(row()+i,column()+j) -= (~rhs)(i,j);
9189  }
9190  }
9191  }
9192  }
9193 }
9195 //*************************************************************************************************
9196 
9197 
9198 //*************************************************************************************************
9210 template< typename MT // Type of the dense matrix
9211  , size_t... CSAs > // Compile time submatrix arguments
9212 template< typename MT2 > // Type of the right-hand side sparse matrix
9213 inline void Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
9214 {
9215  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9216  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9217 
9218  for( size_t i=0UL; i<rows(); ++i )
9219  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
9220  matrix_(row()+i,column()+element->index()) -= element->value();
9221 }
9223 //*************************************************************************************************
9224 
9225 
9226 //*************************************************************************************************
9238 template< typename MT // Type of the dense matrix
9239  , size_t... CSAs > // Compile time submatrix arguments
9240 template< typename MT2 > // Type of the right-hand side sparse matrix
9241 inline void Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
9242 {
9244 
9245  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9246  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9247 
9248  for( size_t j=0UL; j<columns(); ++j )
9249  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
9250  matrix_(row()+element->index(),column()+j) -= element->value();
9251 }
9253 //*************************************************************************************************
9254 
9255 
9256 //*************************************************************************************************
9268 template< typename MT // Type of the dense matrix
9269  , size_t... CSAs > // Compile time submatrix arguments
9270 template< typename MT2 > // Type of the right-hand side dense matrix
9271 inline DisableIf_< typename Submatrix<MT,aligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedSchurAssign<MT2> >
9272  Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
9273 {
9274  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9275  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9276 
9277  const size_t jpos( columns() & size_t(-2) );
9278  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
9279 
9280  for( size_t i=0UL; i<rows(); ++i ) {
9281  for( size_t j=0UL; j<jpos; j+=2UL ) {
9282  matrix_(row()+i,column()+j ) *= (~rhs)(i,j );
9283  matrix_(row()+i,column()+j+1UL) *= (~rhs)(i,j+1UL);
9284  }
9285  if( jpos < columns() ) {
9286  matrix_(row()+i,column()+jpos) *= (~rhs)(i,jpos);
9287  }
9288  }
9289 }
9291 //*************************************************************************************************
9292 
9293 
9294 //*************************************************************************************************
9306 template< typename MT // Type of the dense matrix
9307  , size_t... CSAs > // Compile time submatrix arguments
9308 template< typename MT2 > // Type of the right-hand side dense matrix
9309 inline EnableIf_< typename Submatrix<MT,aligned,false,true,CSAs...>::BLAZE_TEMPLATE VectorizedSchurAssign<MT2> >
9310  Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
9311 {
9313 
9314  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9315  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9316 
9317  for( size_t i=0UL; i<rows(); ++i )
9318  {
9319  const size_t jpos( columns() & size_t(-SIMDSIZE) );
9320  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
9321 
9322  size_t j( 0UL );
9323  Iterator left( begin(i) );
9324  ConstIterator_<MT2> right( (~rhs).begin(i) );
9325 
9326  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
9327  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9328  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9329  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9330  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9331  }
9332  for( ; j<jpos; j+=SIMDSIZE ) {
9333  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9334  }
9335  for( ; j<columns(); ++j ) {
9336  *left *= *right; ++left; ++right;
9337  }
9338  }
9339 }
9341 //*************************************************************************************************
9342 
9343 
9344 //*************************************************************************************************
9356 template< typename MT // Type of the dense matrix
9357  , size_t... CSAs > // Compile time submatrix arguments
9358 template< typename MT2 > // Type of the right-hand side dense matrix
9359 inline void Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
9360 {
9362 
9363  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9364  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9365 
9366  constexpr size_t block( BLOCK_SIZE );
9367 
9368  for( size_t ii=0UL; ii<rows(); ii+=block ) {
9369  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
9370  for( size_t jj=0UL; jj<columns(); jj+=block ) {
9371  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
9372  for( size_t i=ii; i<iend; ++i ) {
9373  for( size_t j=jj; j<jend; ++j ) {
9374  matrix_(row()+i,column()+j) *= (~rhs)(i,j);
9375  }
9376  }
9377  }
9378  }
9379 }
9381 //*************************************************************************************************
9382 
9383 
9384 //*************************************************************************************************
9396 template< typename MT // Type of the dense matrix
9397  , size_t... CSAs > // Compile time submatrix arguments
9398 template< typename MT2 > // Type of the right-hand side sparse matrix
9399 inline void Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
9400 {
9401  using blaze::reset;
9402 
9403  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9404  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9405 
9406  for( size_t i=0UL; i<rows(); ++i )
9407  {
9408  size_t j( 0UL );
9409 
9410  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element ) {
9411  for( ; j<element->index(); ++j )
9412  reset( matrix_(row()+i,column()+j) );
9413  matrix_(row()+i,column()+j) *= element->value();
9414  ++j;
9415  }
9416 
9417  for( ; j<columns(); ++j ) {
9418  reset( matrix_(row()+i,column()+j) );
9419  }
9420  }
9421 }
9423 //*************************************************************************************************
9424 
9425 
9426 //*************************************************************************************************
9438 template< typename MT // Type of the dense matrix
9439  , size_t... CSAs > // Compile time submatrix arguments
9440 template< typename MT2 > // Type of the right-hand side sparse matrix
9441 inline void Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
9442 {
9443  using blaze::reset;
9444 
9446 
9447  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9448  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9449 
9450  for( size_t j=0UL; j<columns(); ++j )
9451  {
9452  size_t i( 0UL );
9453 
9454  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element ) {
9455  for( ; i<element->index(); ++i )
9456  reset( matrix_(row()+i,column()+j) );
9457  matrix_(row()+element->index(),column()+j) *= element->value();
9458  ++i;
9459  }
9460 
9461  for( ; i<rows(); ++i ) {
9462  reset( matrix_(row()+i,column()+j) );
9463  }
9464  }
9465 }
9467 //*************************************************************************************************
9468 
9469 
9470 
9471 
9472 
9473 
9474 
9475 
9476 //=================================================================================================
9477 //
9478 // CLASS TEMPLATE SPECIALIZATION FOR ALIGNED COLUMN-MAJOR DENSE SUBMATRICES
9479 //
9480 //=================================================================================================
9481 
9482 //*************************************************************************************************
9490 template< typename MT // Type of the dense matrix
9491  , size_t... CSAs > // Compile time submatrix arguments
9492 class Submatrix<MT,aligned,true,true,CSAs...>
9493  : public View< DenseMatrix< Submatrix<MT,aligned,true,true,CSAs...>, true > >
9494  , private SubmatrixData<CSAs...>
9495 {
9496  private:
9497  //**Type definitions****************************************************************************
9498  using DataType = SubmatrixData<CSAs...>;
9499  using Operand = If_< IsExpression<MT>, MT, MT& >;
9500  //**********************************************************************************************
9501 
9502  public:
9503  //**Type definitions****************************************************************************
9505  using This = Submatrix<MT,aligned,true,true,CSAs...>;
9506 
9507  using BaseType = DenseMatrix<This,true>;
9508  using ViewedType = MT;
9509  using ResultType = SubmatrixTrait_<MT,CSAs...>;
9510  using OppositeType = OppositeType_<ResultType>;
9511  using TransposeType = TransposeType_<ResultType>;
9512  using ElementType = ElementType_<MT>;
9513  using SIMDType = SIMDTrait_<ElementType>;
9514  using ReturnType = ReturnType_<MT>;
9515  using CompositeType = const Submatrix&;
9516 
9518  using ConstReference = ConstReference_<MT>;
9519 
9521  using Reference = If_< IsConst<MT>, ConstReference, Reference_<MT> >;
9522 
9524  using ConstPointer = ConstPointer_<MT>;
9525 
9527  using Pointer = If_< Or< IsConst<MT>, Not< HasMutableDataAccess<MT> > >, ConstPointer, Pointer_<MT> >;
9528 
9530  using ConstIterator = ConstIterator_<MT>;
9531 
9533  using Iterator = If_< IsConst<MT>, ConstIterator, Iterator_<MT> >;
9534  //**********************************************************************************************
9535 
9536  //**Compilation flags***************************************************************************
9538  enum : bool { simdEnabled = MT::simdEnabled };
9539 
9541  enum : bool { smpAssignable = MT::smpAssignable };
9542  //**********************************************************************************************
9543 
9544  //**Constructors********************************************************************************
9547  template< typename... RSAs >
9548  explicit inline Submatrix( MT& matrix, RSAs... args );
9549  // No explicitly declared copy constructor.
9551  //**********************************************************************************************
9552 
9553  //**Destructor**********************************************************************************
9554  // No explicitly declared destructor.
9555  //**********************************************************************************************
9556 
9557  //**Data access functions***********************************************************************
9560  inline Reference operator()( size_t i, size_t j );
9561  inline ConstReference operator()( size_t i, size_t j ) const;
9562  inline Reference at( size_t i, size_t j );
9563  inline ConstReference at( size_t i, size_t j ) const;
9564  inline Pointer data () noexcept;
9565  inline ConstPointer data () const noexcept;
9566  inline Pointer data ( size_t j ) noexcept;
9567  inline ConstPointer data ( size_t j ) const noexcept;
9568  inline Iterator begin ( size_t j );
9569  inline ConstIterator begin ( size_t j ) const;
9570  inline ConstIterator cbegin( size_t j ) const;
9571  inline Iterator end ( size_t j );
9572  inline ConstIterator end ( size_t j ) const;
9573  inline ConstIterator cend ( size_t j ) const;
9575  //**********************************************************************************************
9576 
9577  //**Assignment operators************************************************************************
9580  inline Submatrix& operator=( const ElementType& rhs );
9581  inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
9582  inline Submatrix& operator=( const Submatrix& rhs );
9583 
9584  template< typename MT2, bool SO >
9585  inline Submatrix& operator=( const Matrix<MT2,SO>& rhs );
9586 
9587  template< typename MT2, bool SO >
9588  inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
9589  operator+=( const Matrix<MT2,SO>& rhs );
9590 
9591  template< typename MT2, bool SO >
9592  inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
9593  operator+=( const Matrix<MT2,SO>& rhs );
9594 
9595  template< typename MT2, bool SO >
9596  inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
9597  operator-=( const Matrix<MT2,SO>& rhs );
9598 
9599  template< typename MT2, bool SO >
9600  inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
9601  operator-=( const Matrix<MT2,SO>& rhs );
9602 
9603  template< typename MT2, bool SO >
9604  inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
9605  operator%=( const Matrix<MT2,SO>& rhs );
9606 
9607  template< typename MT2, bool SO >
9608  inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix& >
9609  operator%=( const Matrix<MT2,SO>& rhs );
9611  //**********************************************************************************************
9612 
9613  //**Utility functions***************************************************************************
9616  using DataType::row;
9617  using DataType::column;
9618  using DataType::rows;
9619  using DataType::columns;
9620 
9621  inline MT& operand() noexcept;
9622  inline const MT& operand() const noexcept;
9623 
9624  inline size_t spacing() const noexcept;
9625  inline size_t capacity() const noexcept;
9626  inline size_t capacity( size_t i ) const noexcept;
9627  inline size_t nonZeros() const;
9628  inline size_t nonZeros( size_t i ) const;
9629  inline void reset();
9630  inline void reset( size_t i );
9632  //**********************************************************************************************
9633 
9634  //**Numeric functions***************************************************************************
9637  inline Submatrix& transpose();
9638  inline Submatrix& ctranspose();
9639 
9640  template< typename Other > inline Submatrix& scale( const Other& scalar );
9642  //**********************************************************************************************
9643 
9644  private:
9645  //**********************************************************************************************
9647  template< typename MT2 >
9648  struct VectorizedAssign {
9649  enum : bool { value = useOptimizedKernels &&
9650  simdEnabled && MT2::simdEnabled &&
9651  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value };
9652  };
9653  //**********************************************************************************************
9654 
9655  //**********************************************************************************************
9657  template< typename MT2 >
9658  struct VectorizedAddAssign {
9659  enum : bool { value = useOptimizedKernels &&
9660  simdEnabled && MT2::simdEnabled &&
9661  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value &&
9662  HasSIMDAdd< ElementType, ElementType_<MT2> >::value &&
9663  !IsDiagonal<MT2>::value };
9664  };
9665  //**********************************************************************************************
9666 
9667  //**********************************************************************************************
9669  template< typename MT2 >
9670  struct VectorizedSubAssign {
9671  enum : bool { value = useOptimizedKernels &&
9672  simdEnabled && MT2::simdEnabled &&
9673  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value &&
9674  HasSIMDSub< ElementType, ElementType_<MT2> >::value &&
9675  !IsDiagonal<MT2>::value };
9676  };
9677  //**********************************************************************************************
9678 
9679  //**********************************************************************************************
9681  template< typename MT2 >
9682  struct VectorizedSchurAssign {
9683  enum : bool { value = useOptimizedKernels &&
9684  simdEnabled && MT2::simdEnabled &&
9685  IsSIMDCombinable< ElementType, ElementType_<MT2> >::value &&
9686  HasSIMDMult< ElementType, ElementType_<MT2> >::value };
9687  };
9688  //**********************************************************************************************
9689 
9690  //**SIMD properties*****************************************************************************
9692  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
9693  //**********************************************************************************************
9694 
9695  public:
9696  //**Expression template evaluation functions****************************************************
9699  template< typename Other >
9700  inline bool canAlias( const Other* alias ) const noexcept;
9701 
9702  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
9703  inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
9704 
9705  template< typename Other >
9706  inline bool isAliased( const Other* alias ) const noexcept;
9707 
9708  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
9709  inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
9710 
9711  inline bool isAligned () const noexcept;
9712  inline bool canSMPAssign() const noexcept;
9713 
9714  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
9715  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
9716  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
9717 
9718  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
9719  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
9720  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
9721  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
9722 
9723  template< typename MT2 >
9724  inline DisableIf_< VectorizedAssign<MT2> > assign( const DenseMatrix<MT2,true>& rhs );
9725 
9726  template< typename MT2 >
9727  inline EnableIf_< VectorizedAssign<MT2> > assign( const DenseMatrix<MT2,true>& rhs );
9728 
9729  template< typename MT2 > inline void assign( const DenseMatrix<MT2,false>& rhs );
9730  template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
9731  template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
9732 
9733  template< typename MT2 >
9734  inline DisableIf_< VectorizedAddAssign<MT2> > addAssign( const DenseMatrix<MT2,true>& rhs );
9735 
9736  template< typename MT2 >
9737  inline EnableIf_< VectorizedAddAssign<MT2> > addAssign( const DenseMatrix<MT2,true>& rhs );
9738 
9739  template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,false>& rhs );
9740  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
9741  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
9742 
9743  template< typename MT2 >
9744  inline DisableIf_< VectorizedSubAssign<MT2> > subAssign( const DenseMatrix<MT2,true>& rhs );
9745 
9746  template< typename MT2 >
9747  inline EnableIf_< VectorizedSubAssign<MT2> > subAssign( const DenseMatrix<MT2,true>& rhs );
9748 
9749  template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,false>& rhs );
9750  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
9751  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
9752 
9753  template< typename MT2 >
9754  inline DisableIf_< VectorizedSchurAssign<MT2> > schurAssign( const DenseMatrix<MT2,true>& rhs );
9755 
9756  template< typename MT2 >
9757  inline EnableIf_< VectorizedSchurAssign<MT2> > schurAssign( const DenseMatrix<MT2,true>& rhs );
9758 
9759  template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,false>& rhs );
9760  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
9761  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
9763  //**********************************************************************************************
9764 
9765  private:
9766  //**Utility functions***************************************************************************
9769  inline bool hasOverlap() const noexcept;
9771  //**********************************************************************************************
9772 
9773  //**Member variables****************************************************************************
9776  Operand matrix_;
9777 
9778  //**********************************************************************************************
9779 
9780  //**Friend declarations*************************************************************************
9781  template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
9782  //**********************************************************************************************
9783 
9784  //**Compile time checks*************************************************************************
9792  //**********************************************************************************************
9793 };
9795 //*************************************************************************************************
9796 
9797 
9798 
9799 
9800 //=================================================================================================
9801 //
9802 // CONSTRUCTORS
9803 //
9804 //=================================================================================================
9805 
9806 //*************************************************************************************************
9819 template< typename MT // Type of the dense matrix
9820  , size_t... CSAs > // Compile time submatrix arguments
9821 template< typename... RSAs > // Runtime submatrix arguments
9822 inline Submatrix<MT,aligned,true,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
9823  : DataType( args... ) // Base class initialization
9824  , matrix_ ( matrix ) // The matrix containing the submatrix
9825 {
9826  if( !Contains< TypeList<RSAs...>, Unchecked >::value )
9827  {
9828  if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
9829  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
9830  }
9831 
9832  if( ( simdEnabled && matrix_.data() != nullptr && !checkAlignment( data() ) ) ||
9833  ( columns() > 1UL && matrix_.spacing() % SIMDSIZE != 0UL ) ) {
9834  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix alignment" );
9835  }
9836  }
9837  else {
9838  BLAZE_INTERNAL_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
9839  BLAZE_INTERNAL_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
9840 
9841  BLAZE_INTERNAL_ASSERT( !simdEnabled || matrix_.data() == nullptr || checkAlignment( data() ), "Invalid submatrix alignment" );
9842  BLAZE_INTERNAL_ASSERT( columns() <= 1UL || matrix_.spacing() % SIMDSIZE == 0UL, "Invalid submatrix alignment" );
9843  }
9844 }
9846 //*************************************************************************************************
9847 
9848 
9849 
9850 
9851 //=================================================================================================
9852 //
9853 // DATA ACCESS FUNCTIONS
9854 //
9855 //=================================================================================================
9856 
9857 //*************************************************************************************************
9868 template< typename MT // Type of the dense matrix
9869  , size_t... CSAs > // Compile time submatrix arguments
9870 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Reference
9871  Submatrix<MT,aligned,true,true,CSAs...>::operator()( size_t i, size_t j )
9872 {
9873  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
9874  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
9875 
9876  return matrix_(row()+i,column()+j);
9877 }
9879 //*************************************************************************************************
9880 
9881 
9882 //*************************************************************************************************
9893 template< typename MT // Type of the dense matrix
9894  , size_t... CSAs > // Compile time submatrix arguments
9895 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstReference
9896  Submatrix<MT,aligned,true,true,CSAs...>::operator()( size_t i, size_t j ) const
9897 {
9898  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
9899  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
9900 
9901  return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
9902 }
9904 //*************************************************************************************************
9905 
9906 
9907 //*************************************************************************************************
9919 template< typename MT // Type of the dense matrix
9920  , size_t... CSAs > // Compile time submatrix arguments
9921 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Reference
9922  Submatrix<MT,aligned,true,true,CSAs...>::at( size_t i, size_t j )
9923 {
9924  if( i >= rows() ) {
9925  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
9926  }
9927  if( j >= columns() ) {
9928  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
9929  }
9930  return (*this)(i,j);
9931 }
9933 //*************************************************************************************************
9934 
9935 
9936 //*************************************************************************************************
9948 template< typename MT // Type of the dense matrix
9949  , size_t... CSAs > // Compile time submatrix arguments
9950 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstReference
9951  Submatrix<MT,aligned,true,true,CSAs...>::at( size_t i, size_t j ) const
9952 {
9953  if( i >= rows() ) {
9954  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
9955  }
9956  if( j >= columns() ) {
9957  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
9958  }
9959  return (*this)(i,j);
9960 }
9962 //*************************************************************************************************
9963 
9964 
9965 //*************************************************************************************************
9975 template< typename MT // Type of the dense matrix
9976  , size_t... CSAs > // Compile time submatrix arguments
9977 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Pointer
9979 {
9980  return matrix_.data() + row() + column()*spacing();
9981 }
9983 //*************************************************************************************************
9984 
9985 
9986 //*************************************************************************************************
9996 template< typename MT // Type of the dense matrix
9997  , size_t... CSAs > // Compile time submatrix arguments
9998 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstPointer
10000 {
10001  return matrix_.data() + row() + column()*spacing();
10002 }
10004 //*************************************************************************************************
10005 
10006 
10007 //*************************************************************************************************
10016 template< typename MT // Type of the dense matrix
10017  , size_t... CSAs > // Compile time submatrix arguments
10018 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Pointer
10020 {
10021  return matrix_.data() + row() + (column()+j)*spacing();
10022 }
10024 //*************************************************************************************************
10025 
10026 
10027 //*************************************************************************************************
10036 template< typename MT // Type of the dense matrix
10037  , size_t... CSAs > // Compile time submatrix arguments
10038 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstPointer
10039  Submatrix<MT,aligned,true,true,CSAs...>::data( size_t j ) const noexcept
10040 {
10041  return matrix_.data() + row() + (column()+j)*spacing();
10042 }
10044 //*************************************************************************************************
10045 
10046 
10047 //*************************************************************************************************
10054 template< typename MT // Type of the dense matrix
10055  , size_t... CSAs > // Compile time submatrix arguments
10056 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Iterator
10058 {
10059  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10060  return ( matrix_.begin( column() + j ) + row() );
10061 }
10063 //*************************************************************************************************
10064 
10065 
10066 //*************************************************************************************************
10073 template< typename MT // Type of the dense matrix
10074  , size_t... CSAs > // Compile time submatrix arguments
10075 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10077 {
10078  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10079  return ( matrix_.cbegin( column() + j ) + row() );
10080 }
10082 //*************************************************************************************************
10083 
10084 
10085 //*************************************************************************************************
10092 template< typename MT // Type of the dense matrix
10093  , size_t... CSAs > // Compile time submatrix arguments
10094 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10096 {
10097  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10098  return ( matrix_.cbegin( column() + j ) + row() );
10099 }
10101 //*************************************************************************************************
10102 
10103 
10104 //*************************************************************************************************
10111 template< typename MT // Type of the dense matrix
10112  , size_t... CSAs > // Compile time submatrix arguments
10113 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Iterator
10115 {
10116  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10117  return ( matrix_.begin( column() + j ) + row() + rows() );
10118 }
10120 //*************************************************************************************************
10121 
10122 
10123 //*************************************************************************************************
10130 template< typename MT // Type of the dense matrix
10131  , size_t... CSAs > // Compile time submatrix arguments
10132 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10134 {
10135  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10136  return ( matrix_.cbegin( column() + j ) + row() + rows() );
10137 }
10139 //*************************************************************************************************
10140 
10141 
10142 //*************************************************************************************************
10149 template< typename MT // Type of the dense matrix
10150  , size_t... CSAs > // Compile time submatrix arguments
10151 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10153 {
10154  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10155  return ( matrix_.cbegin( column() + j ) + row() + rows() );
10156 }
10158 //*************************************************************************************************
10159 
10160 
10161 
10162 
10163 //=================================================================================================
10164 //
10165 // ASSIGNMENT OPERATORS
10166 //
10167 //=================================================================================================
10168 
10169 //*************************************************************************************************
10180 template< typename MT // Type of the dense matrix
10181  , size_t... CSAs > // Compile time submatrix arguments
10182 inline Submatrix<MT,aligned,true,true,CSAs...>&
10183  Submatrix<MT,aligned,true,true,CSAs...>::operator=( const ElementType& rhs )
10184 {
10185  const size_t jend( column() + columns() );
10186  decltype(auto) left( derestrict( matrix_ ) );
10187 
10188  for( size_t j=column(); j<jend; ++j )
10189  {
10190  const size_t ibegin( ( IsLower<MT>::value )
10191  ?( ( IsUniLower<MT>::value || IsStrictlyLower<MT>::value )
10192  ?( max( j+1UL, row() ) )
10193  :( max( j, row() ) ) )
10194  :( row() ) );
10195  const size_t iend ( ( IsUpper<MT>::value )
10196  ?( ( IsUniUpper<MT>::value || IsStrictlyUpper<MT>::value )
10197  ?( min( j, row()+rows() ) )
10198  :( min( j+1UL, row()+rows() ) ) )
10199  :( row()+rows() ) );
10200 
10201  for( size_t i=ibegin; i<iend; ++i ) {
10202  if( !IsRestricted<MT>::value || IsTriangular<MT>::value || trySet( matrix_, i, j, rhs ) )
10203  left(i,j) = rhs;
10204  }
10205  }
10206 
10207  return *this;
10208 }
10210 //*************************************************************************************************
10211 
10212 
10213 //*************************************************************************************************
10229 template< typename MT // Type of the dense matrix
10230  , size_t... CSAs > // Compile time submatrix arguments
10231 inline Submatrix<MT,aligned,true,true,CSAs...>&
10232  Submatrix<MT,aligned,true,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
10233 {
10234  using blaze::reset;
10235 
10236  if( list.size() != rows() ) {
10237  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
10238  }
10239 
10240  if( IsRestricted<MT>::value ) {
10241  const InitializerMatrix<ElementType> tmp( list, columns() );
10242  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10243  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10244  }
10245  }
10246 
10247  decltype(auto) left( derestrict( *this ) );
10248  size_t i( 0UL );
10249 
10250  for( const auto& rowList : list ) {
10251  size_t j( 0UL );
10252  for( const auto& element : rowList ) {
10253  left(i,j) = element;
10254  ++j;
10255  }
10256  for( ; j<columns(); ++j ) {
10257  reset( left(i,j) );
10258  }
10259  ++i;
10260  }
10261 
10262  return *this;
10263 }
10265 //*************************************************************************************************
10266 
10267 
10268 //*************************************************************************************************
10283 template< typename MT // Type of the dense matrix
10284  , size_t... CSAs > // Compile time submatrix arguments
10285 inline Submatrix<MT,aligned,true,true,CSAs...>&
10286  Submatrix<MT,aligned,true,true,CSAs...>::operator=( const Submatrix& rhs )
10287 {
10290 
10291  if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
10292  return *this;
10293 
10294  if( rows() != rhs.rows() || columns() != rhs.columns() ) {
10295  BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
10296  }
10297 
10298  if( !tryAssign( matrix_, rhs, row(), column() ) ) {
10299  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10300  }
10301 
10302  decltype(auto) left( derestrict( *this ) );
10303 
10304  if( rhs.canAlias( &matrix_ ) ) {
10305  const ResultType tmp( rhs );
10306  smpAssign( left, tmp );
10307  }
10308  else {
10309  smpAssign( left, rhs );
10310  }
10311 
10312  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10313 
10314  return *this;
10315 }
10317 //*************************************************************************************************
10318 
10319 
10320 //*************************************************************************************************
10334 template< typename MT // Type of the dense matrix
10335  , size_t... CSAs > // Compile time submatrix arguments
10336 template< typename MT2 // Type of the right-hand side matrix
10337  , bool SO > // Storage order of the right-hand side matrix
10338 inline Submatrix<MT,aligned,true,true,CSAs...>&
10339  Submatrix<MT,aligned,true,true,CSAs...>::operator=( const Matrix<MT2,SO>& rhs )
10340 {
10342 
10343  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10344  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10345  }
10346 
10347  using Right = If_< IsRestricted<MT>, CompositeType_<MT2>, const MT2& >;
10348  Right right( ~rhs );
10349 
10350  if( !tryAssign( matrix_, right, row(), column() ) ) {
10351  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10352  }
10353 
10354  decltype(auto) left( derestrict( *this ) );
10355 
10356  if( IsReference<Right>::value && right.canAlias( &matrix_ ) ) {
10357  const ResultType_<MT2> tmp( right );
10358  if( IsSparseMatrix<MT2>::value )
10359  reset();
10360  smpAssign( left, tmp );
10361  }
10362  else {
10363  if( IsSparseMatrix<MT2>::value )
10364  reset();
10365  smpAssign( left, right );
10366  }
10367 
10368  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10369 
10370  return *this;
10371 }
10373 //*************************************************************************************************
10374 
10375 
10376 //*************************************************************************************************
10390 template< typename MT // Type of the dense matrix
10391  , size_t... CSAs > // Compile time submatrix arguments
10392 template< typename MT2 // Type of the right-hand side matrix
10393  , bool SO > // Storage order of the right-hand side matrix
10394 inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,aligned,true,true,CSAs...>& >
10395  Submatrix<MT,aligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
10396 {
10400 
10401  using AddType = AddTrait_< ResultType, ResultType_<MT2> >;
10402 
10405 
10406  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10407  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10408  }
10409 
10410  if( !tryAddAssign( matrix_, ~rhs, row(), column() ) ) {
10411  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10412  }
10413 
10414  decltype(auto) left( derestrict( *this ) );
10415 
10416  if( ( ( IsSymmetric<MT>::value || IsHermitian<MT>::value ) && hasOverlap() ) ||
10417  (~rhs).canAlias( &matrix_ ) ) {
10418  const AddType tmp( *this + (~rhs) );
10419  smpAssign( left, tmp );
10420  }
10421  else {
10422  smpAddAssign( left, ~rhs );
10423  }
10424 
10425  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10426 
10427  return *this;
10428 }
10430 //*************************************************************************************************
10431 
10432 
10433 //*************************************************************************************************
10447 template< typename MT // Type of the dense matrix
10448  , size_t... CSAs > // Compile time submatrix arguments
10449 template< typename MT2 // Type of the right-hand side matrix
10450  , bool SO > // Storage order of the right-hand side matrix
10451 inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,aligned,true,true,CSAs...>& >
10452  Submatrix<MT,aligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
10453 {
10457 
10458  using AddType = AddTrait_< ResultType, ResultType_<MT2> >;
10459 
10462 
10463  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10464  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10465  }
10466 
10467  const AddType tmp( *this + (~rhs) );
10468 
10469  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10470  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10471  }
10472 
10473  decltype(auto) left( derestrict( *this ) );
10474 
10475  smpAssign( left, tmp );
10476 
10477  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10478 
10479  return *this;
10480 }
10482 //*************************************************************************************************
10483 
10484 
10485 //*************************************************************************************************
10499 template< typename MT // Type of the dense matrix
10500  , size_t... CSAs > // Compile time submatrix arguments
10501 template< typename MT2 // Type of the right-hand side matrix
10502  , bool SO > // Storage order of the right-hand side matrix
10503 inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,aligned,true,true,CSAs...>& >
10504  Submatrix<MT,aligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
10505 {
10509 
10510  using SubType = SubTrait_< ResultType, ResultType_<MT2> >;
10511 
10514 
10515  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10516  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10517  }
10518 
10519  if( !trySubAssign( matrix_, ~rhs, row(), column() ) ) {
10520  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10521  }
10522 
10523  decltype(auto) left( derestrict( *this ) );
10524 
10525  if( ( ( IsSymmetric<MT>::value || IsHermitian<MT>::value ) && hasOverlap() ) ||
10526  (~rhs).canAlias( &matrix_ ) ) {
10527  const SubType tmp( *this - (~rhs ) );
10528  smpAssign( left, tmp );
10529  }
10530  else {
10531  smpSubAssign( left, ~rhs );
10532  }
10533 
10534  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10535 
10536  return *this;
10537 }
10539 //*************************************************************************************************
10540 
10541 
10542 //*************************************************************************************************
10556 template< typename MT // Type of the dense matrix
10557  , size_t... CSAs > // Compile time submatrix arguments
10558 template< typename MT2 // Type of the right-hand side matrix
10559  , bool SO > // Storage order of the right-hand side matrix
10560 inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,aligned,true,true,CSAs...>& >
10561  Submatrix<MT,aligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
10562 {
10566 
10567  using SubType = SubTrait_< ResultType, ResultType_<MT2> >;
10568 
10571 
10572  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10573  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10574  }
10575 
10576  const SubType tmp( *this - (~rhs) );
10577 
10578  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10579  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10580  }
10581 
10582  decltype(auto) left( derestrict( *this ) );
10583 
10584  smpAssign( left, tmp );
10585 
10586  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10587 
10588  return *this;
10589 }
10591 //*************************************************************************************************
10592 
10593 
10594 //*************************************************************************************************
10608 template< typename MT // Type of the dense matrix
10609  , size_t... CSAs > // Compile time submatrix arguments
10610 template< typename MT2 // Type of the right-hand side matrix
10611  , bool SO > // Storage order of the right-hand side matrix
10612 inline DisableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,aligned,true,true,CSAs...>& >
10613  Submatrix<MT,aligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
10614 {
10618 
10619  using SchurType = SchurTrait_< ResultType, ResultType_<MT2> >;
10620 
10622 
10623  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10624  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10625  }
10626 
10627  if( !trySchurAssign( matrix_, ~rhs, row(), column() ) ) {
10628  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10629  }
10630 
10631  decltype(auto) left( derestrict( *this ) );
10632 
10633  if( ( ( IsSymmetric<MT>::value || IsHermitian<MT>::value ) && hasOverlap() ) ||
10634  (~rhs).canAlias( &matrix_ ) ) {
10635  const SchurType tmp( *this % (~rhs) );
10636  if( IsSparseMatrix<SchurType>::value )
10637  reset();
10638  smpAssign( left, tmp );
10639  }
10640  else {
10641  smpSchurAssign( left, ~rhs );
10642  }
10643 
10644  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10645 
10646  return *this;
10647 }
10649 //*************************************************************************************************
10650 
10651 
10652 //*************************************************************************************************
10666 template< typename MT // Type of the dense matrix
10667  , size_t... CSAs > // Compile time submatrix arguments
10668 template< typename MT2 // Type of the right-hand side matrix
10669  , bool SO > // Storage order of the right-hand side matrix
10670 inline EnableIf_< And< IsRestricted<MT>, RequiresEvaluation<MT2> >, Submatrix<MT,aligned,true,true,CSAs...>& >
10671  Submatrix<MT,aligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
10672 {
10676 
10677  using SchurType = SchurTrait_< ResultType, ResultType_<MT2> >;
10678 
10680 
10681  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10682  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10683  }
10684 
10685  const SchurType tmp( *this % (~rhs) );
10686 
10687  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10688  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10689  }
10690 
10691  decltype(auto) left( derestrict( *this ) );
10692 
10693  if( IsSparseMatrix<SchurType>::value ) {
10694  reset();
10695  }
10696 
10697  smpAssign( left, tmp );
10698 
10699  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10700 
10701  return *this;
10702 }
10704 //*************************************************************************************************
10705 
10706 
10707 
10708 
10709 //=================================================================================================
10710 //
10711 // UTILITY FUNCTIONS
10712 //
10713 //=================================================================================================
10714 
10715 //*************************************************************************************************
10721 template< typename MT // Type of the dense matrix
10722  , size_t... CSAs > // Compile time submatrix arguments
10723 inline MT& Submatrix<MT,aligned,true,true,CSAs...>::operand() noexcept
10724 {
10725  return matrix_;
10726 }
10728 //*************************************************************************************************
10729 
10730 
10731 //*************************************************************************************************
10737 template< typename MT // Type of the dense matrix
10738  , size_t... CSAs > // Compile time submatrix arguments
10739 inline const MT& Submatrix<MT,aligned,true,true,CSAs...>::operand() const noexcept
10740 {
10741  return matrix_;
10742 }
10744 //*************************************************************************************************
10745 
10746 
10747 //*************************************************************************************************
10756 template< typename MT // Type of the dense matrix
10757  , size_t... CSAs > // Compile time submatrix arguments
10758 inline size_t Submatrix<MT,aligned,true,true,CSAs...>::spacing() const noexcept
10759 {
10760  return matrix_.spacing();
10761 }
10763 //*************************************************************************************************
10764 
10765 
10766 //*************************************************************************************************
10772 template< typename MT // Type of the dense matrix
10773  , size_t... CSAs > // Compile time submatrix arguments
10774 inline size_t Submatrix<MT,aligned,true,true,CSAs...>::capacity() const noexcept
10775 {
10776  return rows() * columns();
10777 }
10779 //*************************************************************************************************
10780 
10781 
10782 //*************************************************************************************************
10789 template< typename MT // Type of the dense matrix
10790  , size_t... CSAs > // Compile time submatrix arguments
10791 inline size_t Submatrix<MT,aligned,true,true,CSAs...>::capacity( size_t j ) const noexcept
10792 {
10793  UNUSED_PARAMETER( j );
10794 
10795  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
10796 
10797  return rows();
10798 }
10800 //*************************************************************************************************
10801 
10802 
10803 //*************************************************************************************************
10809 template< typename MT // Type of the dense matrix
10810  , size_t... CSAs > // Compile time submatrix arguments
10812 {
10813  const size_t iend( row() + rows() );
10814  const size_t jend( column() + columns() );
10815  size_t nonzeros( 0UL );
10816 
10817  for( size_t j=column(); j<jend; ++j )
10818  for( size_t i=row(); i<iend; ++i )
10819  if( !isDefault( matrix_(i,j) ) )
10820  ++nonzeros;
10821 
10822  return nonzeros;
10823 }
10825 //*************************************************************************************************
10826 
10827 
10828 //*************************************************************************************************
10835 template< typename MT // Type of the dense matrix
10836  , size_t... CSAs > // Compile time submatrix arguments
10837 inline size_t Submatrix<MT,aligned,true,true,CSAs...>::nonZeros( size_t j ) const
10838 {
10839  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
10840 
10841  const size_t iend( row() + rows() );
10842  size_t nonzeros( 0UL );
10843 
10844  for( size_t i=row(); i<iend; ++i )
10845  if( !isDefault( matrix_(i,column()+j) ) )
10846  ++nonzeros;
10847 
10848  return nonzeros;
10849 }
10851 //*************************************************************************************************
10852 
10853 
10854 //*************************************************************************************************
10860 template< typename MT // Type of the dense matrix
10861  , size_t... CSAs > // Compile time submatrix arguments
10863 {
10864  using blaze::clear;
10865 
10866  for( size_t j=column(); j<column()+columns(); ++j )
10867  {
10868  const size_t ibegin( ( IsLower<MT>::value )
10869  ?( ( IsUniLower<MT>::value || IsStrictlyLower<MT>::value )
10870  ?( max( j+1UL, row() ) )
10871  :( max( j, row() ) ) )
10872  :( row() ) );
10873  const size_t iend ( ( IsUpper<MT>::value )
10874  ?( ( IsUniUpper<MT>::value || IsStrictlyUpper<MT>::value )
10875  ?( min( j, row()+rows() ) )
10876  :( min( j+1UL, row()+rows() ) ) )
10877  :( row()+rows() ) );
10878 
10879  for( size_t i=ibegin; i<iend; ++i )
10880  clear( matrix_(i,j) );
10881  }
10882 }
10884 //*************************************************************************************************
10885 
10886 
10887 //*************************************************************************************************
10894 template< typename MT // Type of the dense matrix
10895  , size_t... CSAs > // Compile time submatrix arguments
10896 inline void Submatrix<MT,aligned,true,true,CSAs...>::reset( size_t j )
10897 {
10898  using blaze::clear;
10899 
10900  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
10901 
10902  const size_t ibegin( ( IsLower<MT>::value )
10903  ?( ( IsUniLower<MT>::value || IsStrictlyLower<MT>::value )
10904  ?( max( j+1UL, row() ) )
10905  :( max( j, row() ) ) )
10906  :( row() ) );
10907  const size_t iend ( ( IsUpper<MT>::value )
10908  ?( ( IsUniUpper<MT>::value || IsStrictlyUpper<MT>::value )
10909  ?( min( j, row()+rows() ) )
10910  :( min( j+1UL, row()+rows() ) ) )
10911  :( row()+rows() ) );
10912 
10913  for( size_t i=ibegin; i<iend; ++i )
10914  clear( matrix_(i,column()+j) );
10915 }
10917 //*************************************************************************************************
10918 
10919 
10920 //*************************************************************************************************
10930 template< typename MT // Type of the dense matrix
10931  , size_t... CSAs > // Compile time submatrix arguments
10932 inline bool Submatrix<MT,aligned,true,true,CSAs...>::hasOverlap() const noexcept
10933 {
10934  BLAZE_INTERNAL_ASSERT( IsSymmetric<MT>::value || IsHermitian<MT>::value, "Invalid matrix detected" );
10935 
10936  if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
10937  return false;
10938  else return true;
10939 }
10941 //*************************************************************************************************
10942 
10943 
10944 
10945 
10946 //=================================================================================================
10947 //
10948 // NUMERIC FUNCTIONS
10949 //
10950 //=================================================================================================
10951 
10952 //*************************************************************************************************
10970 template< typename MT // Type of the dense matrix
10971  , size_t... CSAs > // Compile time submatrix arguments
10972 inline Submatrix<MT,aligned,true,true,CSAs...>&
10974 {
10975  if( rows() != columns() ) {
10976  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
10977  }
10978 
10979  if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
10980  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
10981  }
10982 
10983  decltype(auto) left( derestrict( *this ) );
10984  const ResultType tmp( trans( *this ) );
10985 
10986  smpAssign( left, tmp );
10987 
10988  return *this;
10989 }
10991 //*************************************************************************************************
10992 
10993 
10994 //*************************************************************************************************
11012 template< typename MT // Type of the dense matrix
11013  , size_t... CSAs > // Compile time submatrix arguments
11014 inline Submatrix<MT,aligned,true,true,CSAs...>&
11015  Submatrix<MT,aligned,true,true,CSAs...>::ctranspose()
11016 {
11017  if( rows() != columns() ) {
11018  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
11019  }
11020 
11021  if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
11022  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
11023  }
11024 
11025  decltype(auto) left( derestrict( *this ) );
11026  const ResultType tmp( ctrans( *this ) );
11027 
11028  smpAssign( left, tmp );
11029 
11030  return *this;
11031 }
11033 //*************************************************************************************************
11034 
11035 
11036 //*************************************************************************************************
11049 template< typename MT // Type of the dense matrix
11050  , size_t... CSAs > // Compile time submatrix arguments
11051 template< typename Other > // Data type of the scalar value
11052 inline Submatrix<MT,aligned,true,true,CSAs...>&
11053  Submatrix<MT,aligned,true,true,CSAs...>::scale( const Other& scalar )
11054 {
11056 
11057  const size_t jend( column() + columns() );
11058 
11059  for( size_t j=column(); j<jend; ++j )
11060  {
11061  const size_t ibegin( ( IsLower<MT>::value )
11062  ?( ( IsStrictlyLower<MT>::value )
11063  ?( max( j+1UL, row() ) )
11064  :( max( j, row() ) ) )
11065  :( row() ) );
11066  const size_t iend ( ( IsUpper<MT>::value )
11067  ?( ( IsStrictlyUpper<MT>::value )
11068  ?( min( j, row()+rows() ) )
11069  :( min( j+1UL, row()+rows() ) ) )
11070  :( row()+rows() ) );
11071 
11072  for( size_t i=ibegin; i<iend; ++i )
11073  matrix_(i,j) *= scalar;
11074  }
11075 
11076  return *this;
11077 }
11079 //*************************************************************************************************
11080 
11081 
11082 
11083 
11084 //=================================================================================================
11085 //
11086 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
11087 //
11088 //=================================================================================================
11089 
11090 //*************************************************************************************************
11101 template< typename MT // Type of the dense matrix
11102  , size_t... CSAs > // Compile time submatrix arguments
11103 template< typename Other > // Data type of the foreign expression
11104 inline bool Submatrix<MT,aligned,true,true,CSAs...>::canAlias( const Other* alias ) const noexcept
11105 {
11106  return matrix_.isAliased( alias );
11107 }
11109 //*************************************************************************************************
11110 
11111 
11112 //*************************************************************************************************
11123 template< typename MT // Type of the dense matrix
11124  , size_t... CSAs > // Compile time submatrix arguments
11125 template< typename MT2 // Data type of the foreign dense submatrix
11126  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
11127  , bool SO2 // Storage order of the foreign dense submatrix
11128  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
11129 inline bool
11130  Submatrix<MT,aligned,true,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
11131 {
11132  return ( matrix_.isAliased( &alias->matrix_ ) &&
11133  ( row() + rows() > alias->row() ) &&
11134  ( row() < alias->row() + alias->rows() ) &&
11135  ( column() + columns() > alias->column() ) &&
11136  ( column() < alias->column() + alias->columns() ) );
11137 }
11139 //*************************************************************************************************
11140 
11141 
11142 //*************************************************************************************************
11153 template< typename MT // Type of the dense matrix
11154  , size_t... CSAs > // Compile time submatrix arguments
11155 template< typename Other > // Data type of the foreign expression
11156 inline bool Submatrix<MT,aligned,true,true,CSAs...>::isAliased( const Other* alias ) const noexcept
11157 {
11158  return matrix_.isAliased( alias );
11159 }
11161 //*************************************************************************************************
11162 
11163 
11164 //*************************************************************************************************
11175 template< typename MT // Type of the dense matrix
11176  , size_t... CSAs > // Compile time submatrix arguments
11177 template< typename MT2 // Data type of the foreign dense submatrix
11178  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
11179  , bool SO2 // Storage order of the foreign dense submatrix
11180  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
11181 inline bool
11182  Submatrix<MT,aligned,true,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
11183 {
11184  return ( matrix_.isAliased( &alias->matrix_ ) &&
11185  ( row() + rows() > alias->row() ) &&
11186  ( row() < alias->row() + alias->rows() ) &&
11187  ( column() + columns() > alias->column() ) &&
11188  ( column() < alias->column() + alias->columns() ) );
11189 }
11191 //*************************************************************************************************
11192 
11193 
11194 //*************************************************************************************************
11204 template< typename MT // Type of the dense matrix
11205  , size_t... CSAs > // Compile time submatrix arguments
11206 inline bool Submatrix<MT,aligned,true,true,CSAs...>::isAligned() const noexcept
11207 {
11208  return true;
11209 }
11211 //*************************************************************************************************
11212 
11213 
11214 //*************************************************************************************************
11225 template< typename MT // Type of the dense matrix
11226  , size_t... CSAs > // Compile time submatrix arguments
11227 inline bool Submatrix<MT,aligned,true,true,CSAs...>::canSMPAssign() const noexcept
11228 {
11229  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
11230 }
11232 //*************************************************************************************************
11233 
11234 
11235 //*************************************************************************************************
11250 template< typename MT // Type of the dense matrix
11251  , size_t... CSAs > // Compile time submatrix arguments
11252 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,true,true,CSAs...>::SIMDType
11253  Submatrix<MT,aligned,true,true,CSAs...>::load( size_t i, size_t j ) const noexcept
11254 {
11255  return loada( i, j );
11256 }
11258 //*************************************************************************************************
11259 
11260 
11261 //*************************************************************************************************
11276 template< typename MT // Type of the dense matrix
11277  , size_t... CSAs > // Compile time submatrix arguments
11278 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,true,true,CSAs...>::SIMDType
11279  Submatrix<MT,aligned,true,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
11280 {
11282 
11283  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11284  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11285  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11286  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11287 
11288  return matrix_.loada( row()+i, column()+j );
11289 }
11291 //*************************************************************************************************
11292 
11293 
11294 //*************************************************************************************************
11309 template< typename MT // Type of the dense matrix
11310  , size_t... CSAs > // Compile time submatrix arguments
11311 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,true,true,CSAs...>::SIMDType
11312  Submatrix<MT,aligned,true,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
11313 {
11315 
11316  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11317  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11318  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11319  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11320 
11321  return matrix_.loadu( row()+i, column()+j );
11322 }
11324 //*************************************************************************************************
11325 
11326 
11327 //*************************************************************************************************
11343 template< typename MT // Type of the dense matrix
11344  , size_t... CSAs > // Compile time submatrix arguments
11346  Submatrix<MT,aligned,true,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
11347 {
11348  storea( i, j, value );
11349 }
11351 //*************************************************************************************************
11352 
11353 
11354 //*************************************************************************************************
11370 template< typename MT // Type of the dense matrix
11371  , size_t... CSAs > // Compile time submatrix arguments
11373  Submatrix<MT,aligned,true,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
11374 {
11376 
11377  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11378  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11379  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11380  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11381 
11382  matrix_.storea( row()+i, column()+j, value );
11383 }
11385 //*************************************************************************************************
11386 
11387 
11388 //*************************************************************************************************
11404 template< typename MT // Type of the dense matrix
11405  , size_t... CSAs > // Compile time submatrix arguments
11407  Submatrix<MT,aligned,true,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
11408 {
11410 
11411  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11412  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11413  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11414  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11415 
11416  matrix_.storeu( row()+i, column()+j, value );
11417 }
11419 //*************************************************************************************************
11420 
11421 
11422 //*************************************************************************************************
11439 template< typename MT // Type of the dense matrix
11440  , size_t... CSAs > // Compile time submatrix arguments
11442  Submatrix<MT,aligned,true,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
11443 {
11445 
11446  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11447  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11448  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11449  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11450 
11451  matrix_.stream( row()+i, column()+j, value );
11452 }
11454 //*************************************************************************************************
11455 
11456 
11457 //*************************************************************************************************
11469 template< typename MT // Type of the dense matrix
11470  , size_t... CSAs > // Compile time submatrix arguments
11471 template< typename MT2 > // Type of the right-hand side dense matrix
11472 inline DisableIf_< typename Submatrix<MT,aligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedAssign<MT2> >
11473  Submatrix<MT,aligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
11474 {
11475  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11476  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11477 
11478  const size_t ipos( rows() & size_t(-2) );
11479  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
11480 
11481  for( size_t j=0UL; j<columns(); ++j ) {
11482  for( size_t i=0UL; i<ipos; i+=2UL ) {
11483  matrix_(row()+i ,column()+j) = (~rhs)(i ,j);
11484  matrix_(row()+i+1UL,column()+j) = (~rhs)(i+1UL,j);
11485  }
11486  if( ipos < rows() ) {
11487  matrix_(row()+ipos,column()+j) = (~rhs)(ipos,j);
11488  }
11489  }
11490 }
11492 //*************************************************************************************************
11493 
11494 
11495 //*************************************************************************************************
11507 template< typename MT // Type of the dense matrix
11508  , size_t... CSAs > // Compile time submatrix arguments
11509 template< typename MT2 > // Type of the right-hand side dense matrix
11510 inline EnableIf_< typename Submatrix<MT,aligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedAssign<MT2> >
11511  Submatrix<MT,aligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
11512 {
11514 
11515  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11516  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11517 
11518  const size_t ipos( rows() & size_t(-SIMDSIZE) );
11519  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
11520 
11521  if( useStreaming &&
11522  rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
11523  !(~rhs).isAliased( &matrix_ ) )
11524  {
11525  for( size_t j=0UL; j<columns(); ++j )
11526  {
11527  size_t i( 0UL );
11528  Iterator left( begin(j) );
11529  ConstIterator_<MT2> right( (~rhs).begin(j) );
11530 
11531  for( ; i<ipos; i+=SIMDSIZE ) {
11532  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11533  }
11534  for( ; i<rows(); ++i ) {
11535  *left = *right; ++left; ++right;
11536  }
11537  }
11538  }
11539  else
11540  {
11541  for( size_t j=0UL; j<columns(); ++j )
11542  {
11543  size_t i( 0UL );
11544  Iterator left( begin(j) );
11545  ConstIterator_<MT2> right( (~rhs).begin(j) );
11546 
11547  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
11548  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11549  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11550  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11551  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11552  }
11553  for( ; i<ipos; i+=SIMDSIZE ) {
11554  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11555  }
11556  for( ; i<rows(); ++i ) {
11557  *left = *right; ++left; ++right;
11558  }
11559  }
11560  }
11561 }
11563 //*************************************************************************************************
11564 
11565 
11566 //*************************************************************************************************
11578 template< typename MT // Type of the dense matrix
11579  , size_t... CSAs > // Compile time submatrix arguments
11580 template< typename MT2 > // Type of the right-hand side dense matrix
11581 inline void Submatrix<MT,aligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
11582 {
11584 
11585  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11586  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11587 
11588  constexpr size_t block( BLOCK_SIZE );
11589 
11590  for( size_t jj=0UL; jj<columns(); jj+=block ) {
11591  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
11592  for( size_t ii=0UL; ii<rows(); ii+=block ) {
11593  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
11594  for( size_t j=jj; j<jend; ++j ) {
11595  for( size_t i=ii; i<iend; ++i ) {
11596  matrix_(row()+i,column()+j) = (~rhs)(i,j);
11597  }
11598  }
11599  }
11600  }
11601 }
11603 //*************************************************************************************************
11604 
11605 
11606 //*************************************************************************************************
11618 template< typename MT // Type of the dense matrix
11619  , size_t... CSAs > // Compile time submatrix arguments
11620 template< typename MT2 > // Type of the right-hand side sparse matrix
11621 inline void Submatrix<MT,aligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
11622 {
11623  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11624  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11625 
11626  for( size_t j=0UL; j<columns(); ++j )
11627  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
11628  matrix_(row()+element->index(),column()+j) = element->value();
11629 }
11631 //*************************************************************************************************
11632 
11633 
11634 //*************************************************************************************************
11646 template< typename MT // Type of the dense matrix
11647  , size_t... CSAs > // Compile time submatrix arguments
11648 template< typename MT2 > // Type of the right-hand side sparse matrix
11649 inline void Submatrix<MT,aligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
11650 {
11652 
11653  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11654  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11655 
11656  for( size_t i=0UL; i<rows(); ++i )
11657  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
11658  matrix_(row()+i,column()+element->index()) = element->value();
11659 }
11661 //*************************************************************************************************
11662 
11663 
11664 //*************************************************************************************************
11676 template< typename MT // Type of the dense matrix
11677  , size_t... CSAs > // Compile time submatrix arguments
11678 template< typename MT2 > // Type of the right-hand side dense matrix
11679 inline DisableIf_< typename Submatrix<MT,aligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedAddAssign<MT2> >
11680  Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
11681 {
11682  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11683  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11684 
11685  const size_t ipos( rows() & size_t(-2) );
11686  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
11687 
11688  for( size_t j=0UL; j<columns(); ++j )
11689  {
11690  if( IsDiagonal<MT2>::value ) {
11691  matrix_(row()+j,column()+j) += (~rhs)(j,j);
11692  }
11693  else {
11694  for( size_t i=0UL; i<ipos; i+=2UL ) {
11695  matrix_(row()+i ,column()+j) += (~rhs)(i ,j);
11696  matrix_(row()+i+1UL,column()+j) += (~rhs)(i+1UL,j);
11697  }
11698  if( ipos < rows() ) {
11699  matrix_(row()+ipos,column()+j) += (~rhs)(ipos,j);
11700  }
11701  }
11702  }
11703 }
11705 //*************************************************************************************************
11706 
11707 
11708 //*************************************************************************************************
11720 template< typename MT // Type of the dense matrix
11721  , size_t... CSAs > // Compile time submatrix arguments
11722 template< typename MT2 > // Type of the right-hand side dense matrix
11723 inline EnableIf_< typename Submatrix<MT,aligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedAddAssign<MT2> >
11724  Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
11725 {
11727 
11728  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11729  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11730 
11731  for( size_t j=0UL; j<columns(); ++j )
11732  {
11733  const size_t ibegin( ( IsLower<MT>::value )
11734  ?( ( IsStrictlyLower<MT>::value ? j+1UL : j ) & size_t(-SIMDSIZE) )
11735  :( 0UL ) );
11736  const size_t iend ( ( IsUpper<MT>::value )
11737  ?( IsStrictlyUpper<MT>::value ? j : j+1UL )
11738  :( rows() ) );
11739  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
11740 
11741  const size_t ipos( iend & size_t(-SIMDSIZE) );
11742  BLAZE_INTERNAL_ASSERT( ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
11743 
11744  size_t i( ibegin );
11745  Iterator left( begin(j) + ibegin );
11746  ConstIterator_<MT2> right( (~rhs).begin(j) + ibegin );
11747 
11748  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
11749  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11750  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11751  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11752  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11753  }
11754  for( ; i<ipos; i+=SIMDSIZE ) {
11755  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11756  }
11757  for( ; i<iend; ++i ) {
11758  *left += *right; ++left; ++right;
11759  }
11760  }
11761 }
11763 //*************************************************************************************************
11764 
11765 
11766 //*************************************************************************************************
11778 template< typename MT // Type of the dense matrix
11779  , size_t... CSAs > // Compile time submatrix arguments
11780 template< typename MT2 > // Type of the right-hand side dense matrix
11781 inline void Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
11782 {
11784 
11785  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11786  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11787 
11788  constexpr size_t block( BLOCK_SIZE );
11789 
11790  for( size_t jj=0UL; jj<columns(); jj+=block ) {
11791  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
11792  for( size_t ii=0UL; ii<rows(); ii+=block ) {
11793  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
11794  for( size_t j=jj; j<jend; ++j ) {
11795  for( size_t i=ii; i<iend; ++i ) {
11796  matrix_(row()+i,column()+j) += (~rhs)(i,j);
11797  }
11798  }
11799  }
11800  }
11801 }
11803 //*************************************************************************************************
11804 
11805 
11806 //*************************************************************************************************
11818 template< typename MT // Type of the dense matrix
11819  , size_t... CSAs > // Compile time submatrix arguments
11820 template< typename MT2 > // Type of the right-hand side sparse matrix
11821 inline void Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
11822 {
11823  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11824  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11825 
11826  for( size_t j=0UL; j<columns(); ++j )
11827  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
11828  matrix_(row()+element->index(),column()+j) += element->value();
11829 }
11831 //*************************************************************************************************
11832 
11833 
11834 //*************************************************************************************************
11846 template< typename MT // Type of the dense matrix
11847  , size_t... CSAs > // Compile time submatrix arguments
11848 template< typename MT2 > // Type of the right-hand side sparse matrix
11849 inline void Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
11850 {
11852 
11853  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11854  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11855 
11856  for( size_t i=0UL; i<rows(); ++i )
11857  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
11858  matrix_(row()+i,column()+element->index()) += element->value();
11859 }
11861 //*************************************************************************************************
11862 
11863 
11864 //*************************************************************************************************
11876 template< typename MT // Type of the dense matrix
11877  , size_t... CSAs > // Compile time submatrix arguments
11878 template< typename MT2 > // Type of the right-hand side dense matrix
11879 inline DisableIf_< typename Submatrix<MT,aligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedSubAssign<MT2> >
11880  Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
11881 {
11882  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11883  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11884 
11885  const size_t ipos( rows() & size_t(-2) );
11886  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
11887 
11888  for( size_t j=0UL; j<columns(); ++j )
11889  {
11890  if( IsDiagonal<MT2>::value ) {
11891  matrix_(row()+j,column()+j) -= (~rhs)(j,j);
11892  }
11893  else {
11894  for( size_t i=0UL; i<ipos; i+=2UL ) {
11895  matrix_(row()+i ,column()+j) -= (~rhs)(i ,j);
11896  matrix_(row()+i+1UL,column()+j) -= (~rhs)(i+1UL,j);
11897  }
11898  if( ipos < rows() ) {
11899  matrix_(row()+ipos,column()+j) -= (~rhs)(ipos,j);
11900  }
11901  }
11902  }
11903 }
11905 //*************************************************************************************************
11906 
11907 
11908 //*************************************************************************************************
11920 template< typename MT // Type of the dense matrix
11921  , size_t... CSAs > // Compile time submatrix arguments
11922 template< typename MT2 > // Type of the right-hand side dense matrix
11923 inline EnableIf_< typename Submatrix<MT,aligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedSubAssign<MT2> >
11924  Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
11925 {
11927 
11928  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11929  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11930 
11931  for( size_t j=0UL; j<columns(); ++j )
11932  {
11933  const size_t ibegin( ( IsLower<MT>::value )
11934  ?( ( IsStrictlyLower<MT>::value ? j+1UL : j ) & size_t(-SIMDSIZE) )
11935  :( 0UL ) );
11936  const size_t iend ( ( IsUpper<MT>::value )
11937  ?( IsStrictlyUpper<MT>::value ? j : j+1UL )
11938  :( rows() ) );
11939  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
11940 
11941  const size_t ipos( iend & size_t(-SIMDSIZE) );
11942  BLAZE_INTERNAL_ASSERT( ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
11943 
11944  size_t i( ibegin );
11945  Iterator left( begin(j) + ibegin );
11946  ConstIterator_<MT2> right( (~rhs).begin(j) + ibegin );
11947 
11948  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
11949  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11950  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11951  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11952  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11953  }
11954  for( ; i<ipos; i+=SIMDSIZE ) {
11955  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11956  }
11957  for( ; i<iend; ++i ) {
11958  *left -= *right; ++left; ++right;
11959  }
11960  }
11961 }
11963 //*************************************************************************************************
11964 
11965 
11966 //*************************************************************************************************
11978 template< typename MT // Type of the dense matrix
11979  , size_t... CSAs > // Compile time submatrix arguments
11980 template< typename MT2 > // Type of the right-hand side dense matrix
11981 inline void Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
11982 {
11984 
11985  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11986  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11987 
11988  constexpr size_t block( BLOCK_SIZE );
11989 
11990  for( size_t jj=0UL; jj<columns(); jj+=block ) {
11991  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
11992  for( size_t ii=0UL; ii<rows(); ii+=block ) {
11993  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
11994  for( size_t j=jj; j<jend; ++j ) {
11995  for( size_t i=ii; i<iend; ++i ) {
11996  matrix_(row()+i,column()+j) -= (~rhs)(i,j);
11997  }
11998  }
11999  }
12000  }
12001 }
12003 //*************************************************************************************************
12004 
12005 
12006 //*************************************************************************************************
12018 template< typename MT // Type of the dense matrix
12019  , size_t... CSAs > // Compile time submatrix arguments
12020 template< typename MT2 > // Type of the right-hand side sparse matrix
12021 inline void Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
12022 {
12023  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12024  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12025 
12026  for( size_t j=0UL; j<columns(); ++j )
12027  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
12028  matrix_(row()+element->index(),column()+j) -= element->value();
12029 }
12031 //*************************************************************************************************
12032 
12033 
12034 //*************************************************************************************************
12046 template< typename MT // Type of the dense matrix
12047  , size_t... CSAs > // Compile time submatrix arguments
12048 template< typename MT2 > // Type of the right-hand side sparse matrix
12049 inline void Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
12050 {
12052 
12053  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12054  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12055 
12056  for( size_t i=0UL; i<rows(); ++i )
12057  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
12058  matrix_(row()+i,column()+element->index()) -= element->value();
12059 }
12061 //*************************************************************************************************
12062 
12063 
12064 //*************************************************************************************************
12076 template< typename MT // Type of the dense matrix
12077  , size_t... CSAs > // Compile time submatrix arguments
12078 template< typename MT2 > // Type of the right-hand side dense matrix
12079 inline DisableIf_< typename Submatrix<MT,aligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedSchurAssign<MT2> >
12080  Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
12081 {
12082  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12083  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12084 
12085  const size_t ipos( rows() & size_t(-2) );
12086  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
12087 
12088  for( size_t j=0UL; j<columns(); ++j ) {
12089  for( size_t i=0UL; i<ipos; i+=2UL ) {
12090  matrix_(row()+i ,column()+j) *= (~rhs)(i ,j);
12091  matrix_(row()+i+1UL,column()+j) *= (~rhs)(i+1UL,j);
12092  }
12093  if( ipos < rows() ) {
12094  matrix_(row()+ipos,column()+j) *= (~rhs)(ipos,j);
12095  }
12096  }
12097 }
12099 //*************************************************************************************************
12100 
12101 
12102 //*************************************************************************************************
12115 template< typename MT // Type of the dense matrix
12116  , size_t... CSAs > // Compile time submatrix arguments
12117 template< typename MT2 > // Type of the right-hand side dense matrix
12118 inline EnableIf_< typename Submatrix<MT,aligned,true,true,CSAs...>::BLAZE_TEMPLATE VectorizedSchurAssign<MT2> >
12119  Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
12120 {
12122 
12123  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12124  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12125 
12126  for( size_t j=0UL; j<columns(); ++j )
12127  {
12128  const size_t ipos( rows() & size_t(-SIMDSIZE) );
12129  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
12130 
12131  size_t i( 0UL );
12132  Iterator left( begin(j) );
12133  ConstIterator_<MT2> right( (~rhs).begin(j) );
12134 
12135  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
12136  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12137  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12138  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12139  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12140  }
12141  for( ; i<ipos; i+=SIMDSIZE ) {
12142  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12143  }
12144  for( ; i<rows(); ++i ) {
12145  *left *= *right; ++left; ++right;
12146  }
12147  }
12148 }
12150 //*************************************************************************************************
12151 
12152 
12153 //*************************************************************************************************
12165 template< typename MT // Type of the dense matrix
12166  , size_t... CSAs > // Compile time submatrix arguments
12167 template< typename MT2 > // Type of the right-hand side dense matrix
12168 inline void Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
12169 {
12171 
12172  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12173  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12174 
12175  constexpr size_t block( BLOCK_SIZE );
12176 
12177  for( size_t jj=0UL; jj<columns(); jj+=block ) {
12178  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
12179  for( size_t ii=0UL; ii<rows(); ii+=block ) {
12180  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
12181  for( size_t j=jj; j<jend; ++j ) {
12182  for( size_t i=ii; i<iend; ++i ) {
12183  matrix_(row()+i,column()+j) *= (~rhs)(i,j);
12184  }
12185  }
12186  }
12187  }
12188 }
12190 //*************************************************************************************************
12191 
12192 
12193 //*************************************************************************************************
12205 template< typename MT // Type of the dense matrix
12206  , size_t... CSAs > // Compile time submatrix arguments
12207 template< typename MT2 > // Type of the right-hand side sparse matrix
12208 inline void Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
12209 {
12210  using blaze::reset;
12211 
12212  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12213  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12214 
12215  for( size_t j=0UL; j<columns(); ++j )
12216  {
12217  size_t i( 0UL );
12218 
12219  for( ConstIterator_<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element ) {
12220  for( ; i<element->index(); ++i )
12221  reset( matrix_(row()+i,column()+j) );
12222  matrix_(row()+i,column()+j) *= element->value();
12223  ++i;
12224  }
12225 
12226  for( ; i<rows(); ++i ) {
12227  reset( matrix_(row()+i,column()+j) );
12228  }
12229  }
12230 }
12232 //*************************************************************************************************
12233 
12234 
12235 //*************************************************************************************************
12247 template< typename MT // Type of the dense matrix
12248  , size_t... CSAs > // Compile time submatrix arguments
12249 template< typename MT2 > // Type of the right-hand side sparse matrix
12250 inline void Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
12251 {
12252  using blaze::reset;
12253 
12255 
12256  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12257  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12258 
12259  for( size_t i=0UL; i<rows(); ++i )
12260  {
12261  size_t j( 0UL );
12262 
12263  for( ConstIterator_<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element ) {
12264  for( ; j<element->index(); ++j )
12265  reset( matrix_(row()+i,column()+j) );
12266  matrix_(row()+i,column()+j) *= element->value();
12267  ++j;
12268  }
12269 
12270  for( ; j<columns(); ++j ) {
12271  reset( matrix_(row()+i,column()+j) );
12272  }
12273  }
12274 }
12276 //*************************************************************************************************
12277 
12278 } // namespace blaze
12279 
12280 #endif
AlignmentFlag
Alignment flag for (un-)aligned vectors and matrices.Via these flags it is possible to specify subvec...
Definition: AlignmentFlag.h:62
Constraint on the data type.
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Constraint on the data type.
Header file for auxiliary alias declarations.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:131
Headerfile for the generic min algorithm.
Header file for the blaze::checked and blaze::unchecked instances.
Header file for kernel specific block sizes.
CompressedMatrix< Type, false > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: CompressedMatrix.h:3077
Header file for the Schur product trait.
#define BLAZE_USER_ASSERT(expr, msg)
Run time assertion macro for user checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_USER_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERT flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:117
Header file for the alignment flag values.
Header file for the UNUSED_PARAMETER function template.
Header file for the IsUniUpper type trait.
EnableIf_< IsDenseMatrix< MT1 > > smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:196
Header file for the subtraction trait.
BLAZE_ALWAYS_INLINE size_t capacity(const Matrix< MT, SO > &matrix) noexcept
Returns the maximum capacity of the matrix.
Definition: Matrix.h:522
Header file for basic type definitions.
Header file for the View base class.
BLAZE_ALWAYS_INLINE EnableIf_< And< IsIntegral< T1 >, HasSize< T1, 1UL > > > storea(T1 *address, const SIMDi8< T2 > &value) noexcept
Aligned store of a vector of 1-byte integral values.
Definition: Storea.h:79
EnableIf_< IsDenseMatrix< MT1 > > smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:164
Header file for the IsSparseMatrix type trait.
Header file for the IsDiagonal type trait.
BLAZE_ALWAYS_INLINE size_t size(const Vector< VT, TF > &vector) noexcept
Returns the current size/dimension of the vector.
Definition: Vector.h:265
#define BLAZE_CONSTRAINT_MUST_NOT_BE_COMPUTATION_TYPE(T)
Constraint on the data type.In case the given data type T is a computational expression (i...
Definition: Computation.h:81
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
constexpr bool operator<(const NegativeAccuracy< A > &lhs, const T &rhs)
Less-than comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:329
This ResultType
Result type for expression template evaluations.
Definition: CompressedMatrix.h:3076
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:364
CompressedMatrix< Type, true > This
Type of this CompressedMatrix instance.
Definition: CompressedMatrix.h:3074
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:588
const DenseIterator< Type, AF > operator+(const DenseIterator< Type, AF > &it, ptrdiff_t inc) noexcept
Addition between a DenseIterator and an integral value.
Definition: DenseIterator.h:701
typename DisableIf< Condition, T >::Type DisableIf_
Auxiliary type for the DisableIf class template.The DisableIf_ alias declaration provides a convenien...
Definition: DisableIf.h:224
CompressedMatrix< Type, false > TransposeType
Transpose type for expression template evaluations.
Definition: CompressedMatrix.h:3078
Header file for the And class template.
const ElementType_< MT > min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1903
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SUBMATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a submatrix type (i.e. a dense or sparse submatrix), a compilation error is created.
Definition: Submatrix.h:81
BLAZE_ALWAYS_INLINE void ctranspose(Matrix< MT, SO > &matrix)
In-place conjugate transpose of the given matrix.
Definition: Matrix.h:827
#define BLAZE_CONSTRAINT_MUST_NOT_BE_UNITRIANGULAR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a lower or upper unitriangular matrix ty...
Definition: UniTriangular.h:81
const Type & ConstReference
Reference to a constant matrix value.
Definition: CompressedMatrix.h:3083
BLAZE_ALWAYS_INLINE size_t nonZeros(const Matrix< MT, SO > &matrix)
Returns the total number of non-zero elements in the matrix.
Definition: Matrix.h:560
const DenseIterator< Type, AF > operator-(const DenseIterator< Type, AF > &it, ptrdiff_t inc) noexcept
Subtraction between a DenseIterator and an integral value.
Definition: DenseIterator.h:733
Element * Iterator
Iterator over non-constant elements.
Definition: CompressedMatrix.h:3084
Constraints on the storage order of matrix types.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_TRANSEXPR_TYPE(T)
Constraint on the data type.In case the given data type T is a transposition expression (i...
Definition: TransExpr.h:81
Header file for the RequiresEvaluation type trait.
Header file for the extended initializer_list functionality.
System settings for performance optimizations.
typename SubmatrixTrait< MT, CSAs... >::Type SubmatrixTrait_
Auxiliary alias declaration for the SubmatrixTrait type trait.The SubmatrixTrait_ alias declaration p...
Definition: SubmatrixTrait.h:145
BLAZE_ALWAYS_INLINE const EnableIf_< And< IsIntegral< T >, HasSize< T, 1UL > >, If_< IsSigned< T >, SIMDint8, SIMDuint8 > > loadu(const T *address) noexcept
Loads a vector of 1-byte integral values.
Definition: Loadu.h:77
Header file for the IsUniLower type trait.
const ElementType_< MT > max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1950
EnableIf_< IsDenseMatrix< MT1 > > smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:133
BLAZE_ALWAYS_INLINE MT::ConstIterator cend(const Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:474
BLAZE_ALWAYS_INLINE MT::ConstIterator cbegin(const Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:408
Constraint on the data type.
BLAZE_ALWAYS_INLINE size_t spacing(const DenseMatrix< MT, SO > &dm) noexcept
Returns the spacing between the beginning of two rows/columns.
Definition: DenseMatrix.h:252
Header file for the matrix storage order types.
Constraint on the data type.
Header file for the implementation of a matrix representation of an initializer list.
Headerfile for the generic max algorithm.
Header file for the DisableIf class template.
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Header file for the IsSymmetric type trait.
MatrixAccessProxy< This > Reference
Reference to a non-constant matrix value.
Definition: CompressedMatrix.h:3082
Header file for the clear shim.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:58
#define BLAZE_ALWAYS_INLINE
Platform dependent setup of an enforced inline keyword.
Definition: Inline.h:85
decltype(auto) ctrans(const DenseMatrix< MT, SO > &dm)
Returns the conjugate transpose matrix of dm.
Definition: DMatMapExpr.h:1359
Header file for nested template disabiguation.
Header file for the If class template.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
SparseMatrix< This, true > BaseType
Base type of this CompressedMatrix instance.
Definition: CompressedMatrix.h:3075
#define BLAZE_CONSTRAINT_MUST_NOT_BE_POINTER_TYPE(T)
Constraint on the data type.In case the given data type T is not a pointer type, a compilation error ...
Definition: Pointer.h:79
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:102
Type ElementType
Type of the compressed matrix elements.
Definition: CompressedMatrix.h:3079
Header file for the Or class template.
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the implementation of the Submatrix base template.
Header file for the HasSIMDAdd type trait.
Header file for the DenseMatrix base class.
constexpr bool operator>(const NegativeAccuracy< A > &lhs, const T &rhs)
Greater-than comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:367
constexpr bool operator>=(const NegativeAccuracy< A > &, const T &rhs)
Greater-or-equal-than comparison between a NegativeAccuracy object and a floating point value...
Definition: Accuracy.h:443
Header file for the Not class template.
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:3085
constexpr bool operator==(const NegativeAccuracy< A > &lhs, const T &rhs)
Equality comparison between a NegativeAccuracy object and a floating point value. ...
Definition: Accuracy.h:250
Header file for all SIMD functionality.
Header file for the IsLower type trait.
BLAZE_ALWAYS_INLINE size_t columns(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of columns of the matrix.
Definition: Matrix.h:506
BLAZE_ALWAYS_INLINE const EnableIf_< And< IsIntegral< T >, HasSize< T, 1UL > >, If_< IsSigned< T >, SIMDint8, SIMDuint8 > > loada(const T *address) noexcept
Loads a vector of 1-byte integral values.
Definition: Loada.h:80
Constraint on the data type.
Header file for the IsTriangular type trait.
#define BLAZE_CONSTRAINT_MUST_BE_VECTORIZABLE_TYPE(T)
Constraint on the data type.In case the given data type T is not a vectorizable data type...
Definition: Vectorizable.h:61
Header file for the exception macros of the math module.
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:430
Header file for the implementation of the SubmatrixData class template.
decltype(auto) operator*(const DenseMatrix< MT1, false > &lhs, const DenseMatrix< MT2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:8893
constexpr bool operator!=(const NegativeAccuracy< A > &lhs, const T &rhs)
Inequality comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:290
Constraint on the data type.
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
void clear(const DiagonalProxy< MT > &proxy)
Clearing the represented element.
Definition: DiagonalProxy.h:608
Constraint on the data type.
Flag for aligned vectors and matrices.
Definition: AlignmentFlag.h:65
BLAZE_ALWAYS_INLINE EnableIf_< And< IsIntegral< T1 >, HasSize< T1, 1UL > > > stream(T1 *address, const SIMDi8< T2 > &value) noexcept
Aligned, non-temporal store of a vector of 1-byte integral values.
Definition: Stream.h:75
Header file for the IsSIMDCombinable type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type, a compilation error is created.
Definition: Symmetric.h:79
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
Header file for the IsConst type trait.
BLAZE_ALWAYS_INLINE T1 & operator+=(SIMDPack< T1 > &lhs, const SIMDPack< T2 > &rhs)
Addition assignment operator for the addition of two SIMD packs.
Definition: BasicTypes.h:1357
Header file for run time assertion macros.
Header file for the addition trait.
Header file for the Unique class template.
Header file for the submatrix trait.
Check< false > Unchecked
Type of the blaze::unchecked instance.blaze::Unchecked is the type of the blaze::unchecked instance...
Definition: Check.h:96
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:131
Header file for the reset shim.
Header file for the cache size of the target architecture.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:79
Header file for the isDefault shim.
Constraint on the data type.
Constraint on the data type.
Header file for the HasSIMDSub type trait.
Constraints on the storage order of matrix types.
constexpr bool operator<=(const NegativeAccuracy< A > &, const T &rhs)
Less-or-equal-than comparison between a NegativeAccuracy object and a floating point value...
Definition: Accuracy.h:405
Header file for the HasMutableDataAccess type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
Header file for the IsReference type trait.
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:224
BLAZE_ALWAYS_INLINE EnableIf_< And< IsIntegral< T1 >, HasSize< T1, 1UL > > > storeu(T1 *address, const SIMDi8< T2 > &value) noexcept
Unaligned store of a vector of 1-byte integral values.
Definition: Storeu.h:76
BLAZE_ALWAYS_INLINE bool checkAlignment(const T *address)
Checks the alignment of the given address.
Definition: AlignmentCheck.h:68
BLAZE_ALWAYS_INLINE size_t rows(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of rows of the matrix.
Definition: Matrix.h:490
const Type & ReturnType
Return type for expression template evaluations.
Definition: CompressedMatrix.h:3080
BLAZE_ALWAYS_INLINE MT::ElementType * data(DenseMatrix< MT, SO > &dm) noexcept
Low-level data access to the dense matrix elements.
Definition: DenseMatrix.h:169
decltype(auto) trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:789
Flag for unaligned vectors and matrices.
Definition: AlignmentFlag.h:64
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:3081
Header file for the alignment check function.
bool isIntact(const DiagonalMatrix< MT, SO, DF > &m)
Returns whether the invariants of the given diagonal matrix are intact.
Definition: DiagonalMatrix.h:254
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:628
#define BLAZE_THROW_LOGIC_ERROR(MESSAGE)
Macro for the emission of a std::logic_error exception.This macro encapsulates the default way of Bla...
Definition: Exception.h:187
BLAZE_ALWAYS_INLINE T1 & operator-=(SIMDPack< T1 > &lhs, const SIMDPack< T2 > &rhs)
Subtraction assignment operator for the subtraction of two SIMD packs.
Definition: BasicTypes.h:1375
Header file for the IsUpper type trait.
void UNUSED_PARAMETER(const Args &...)
Suppression of unused parameter warnings.
Definition: Unused.h:81
Header file for the IsHermitian type trait.
Header file for the IsRestricted type trait.
System settings for the inline keywords.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the IsExpression type trait class.
Constraint on the data type.
BLAZE_ALWAYS_INLINE void transpose(Matrix< MT, SO > &matrix)
In-place transpose of the given matrix.
Definition: Matrix.h:801