Blaze  3.6
CustomMatrix.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_DENSE_CUSTOMMATRIX_H_
36 #define _BLAZE_MATH_DENSE_CUSTOMMATRIX_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <algorithm>
44 #include <utility>
45 #include <blaze/math/Aliases.h>
50 #include <blaze/math/Exception.h>
53 #include <blaze/math/Forward.h>
57 #include <blaze/math/PaddingFlag.h>
58 #include <blaze/math/shims/Clear.h>
63 #include <blaze/math/SIMD.h>
81 #include <blaze/system/Blocking.h>
82 #include <blaze/system/CacheSize.h>
83 #include <blaze/system/Inline.h>
90 #include <blaze/util/Assert.h>
96 #include <blaze/util/DisableIf.h>
97 #include <blaze/util/EnableIf.h>
99 #include <blaze/util/MaybeUnused.h>
100 #include <blaze/util/Misalignment.h>
101 #include <blaze/util/mpl/If.h>
102 #include <blaze/util/Types.h>
109 
110 
111 namespace blaze {
112 
113 //=================================================================================================
114 //
115 // CLASS DEFINITION
116 //
117 //=================================================================================================
118 
119 //*************************************************************************************************
401 template< typename Type // Data type of the matrix
402  , bool AF // Alignment flag
403  , bool PF // Padding flag
404  , bool SO = defaultStorageOrder // Storage order
405  , typename RT = DynamicMatrix<RemoveConst_t<Type>,SO> > // Result type
407  : public DenseMatrix< CustomMatrix<Type,AF,PF,SO,RT>, SO >
408 {
409  public:
410  //**Type definitions****************************************************************************
413 
415  using ResultType = RT;
416 
419 
422 
423  using ElementType = Type;
425  using ReturnType = const Type&;
426  using CompositeType = const This&;
427 
428  using Reference = Type&;
429  using ConstReference = const Type&;
430  using Pointer = Type*;
431  using ConstPointer = const Type*;
432 
435  //**********************************************************************************************
436 
437  //**Rebind struct definition********************************************************************
440  template< typename NewType > // Data type of the other matrix
441  struct Rebind {
444  };
445  //**********************************************************************************************
446 
447  //**Resize struct definition********************************************************************
450  template< size_t NewM // Number of rows of the other matrix
451  , size_t NewN > // Number of columns of the other matrix
452  struct Resize {
455  };
456  //**********************************************************************************************
457 
458  //**Compilation flags***************************************************************************
460 
464  static constexpr bool simdEnabled = IsVectorizable_v<Type>;
465 
467 
470  static constexpr bool smpAssignable = !IsSMPAssignable_v<Type>;
471  //**********************************************************************************************
472 
473  //**Constructors********************************************************************************
476  explicit inline CustomMatrix();
477  explicit inline CustomMatrix( Type* ptr, size_t m, size_t n );
478  explicit inline CustomMatrix( Type* ptr, size_t m, size_t n, size_t nn );
479 
480  inline CustomMatrix( const CustomMatrix& m );
481  inline CustomMatrix( CustomMatrix&& m ) noexcept;
483  //**********************************************************************************************
484 
485  //**Destructor**********************************************************************************
488  ~CustomMatrix() = default;
490  //**********************************************************************************************
491 
492  //**Data access functions***********************************************************************
495  inline Reference operator()( size_t i, size_t j ) noexcept;
496  inline ConstReference operator()( size_t i, size_t j ) const noexcept;
497  inline Reference at( size_t i, size_t j );
498  inline ConstReference at( size_t i, size_t j ) const;
499  inline Pointer data () noexcept;
500  inline ConstPointer data () const noexcept;
501  inline Pointer data ( size_t i ) noexcept;
502  inline ConstPointer data ( size_t i ) const noexcept;
503  inline Iterator begin ( size_t i ) noexcept;
504  inline ConstIterator begin ( size_t i ) const noexcept;
505  inline ConstIterator cbegin( size_t i ) const noexcept;
506  inline Iterator end ( size_t i ) noexcept;
507  inline ConstIterator end ( size_t i ) const noexcept;
508  inline ConstIterator cend ( size_t i ) const noexcept;
510  //**********************************************************************************************
511 
512  //**Assignment operators************************************************************************
515  inline CustomMatrix& operator=( const Type& set );
516  inline CustomMatrix& operator=( initializer_list< initializer_list<Type> > list );
517 
518  template< typename Other, size_t Rows, size_t Cols >
519  inline CustomMatrix& operator=( const Other (&array)[Rows][Cols] );
520 
521  inline CustomMatrix& operator=( const CustomMatrix& rhs );
522  inline CustomMatrix& operator=( CustomMatrix&& rhs ) noexcept;
523 
524  template< typename MT, bool SO2 > inline CustomMatrix& operator= ( const Matrix<MT,SO2>& rhs );
525  template< typename MT, bool SO2 > inline CustomMatrix& operator+=( const Matrix<MT,SO2>& rhs );
526  template< typename MT, bool SO2 > inline CustomMatrix& operator-=( const Matrix<MT,SO2>& rhs );
527  template< typename MT, bool SO2 > inline CustomMatrix& operator%=( const Matrix<MT,SO2>& rhs );
529  //**********************************************************************************************
530 
531  //**Utility functions***************************************************************************
534  inline size_t rows() const noexcept;
535  inline size_t columns() const noexcept;
536  inline size_t spacing() const noexcept;
537  inline size_t capacity() const noexcept;
538  inline size_t capacity( size_t i ) const noexcept;
539  inline size_t nonZeros() const;
540  inline size_t nonZeros( size_t i ) const;
541  inline void reset();
542  inline void reset( size_t i );
543  inline void clear();
544  inline void swap( CustomMatrix& m ) noexcept;
546  //**********************************************************************************************
547 
548  //**Numeric functions***************************************************************************
551  inline CustomMatrix& transpose();
552  inline CustomMatrix& ctranspose();
553 
554  template< typename Other > inline CustomMatrix& scale( const Other& scalar );
556  //**********************************************************************************************
557 
558  //**Resource management functions***************************************************************
561  inline void reset( Type* ptr, size_t m, size_t n );
562  inline void reset( Type* ptr, size_t m, size_t n, size_t nn );
564  //**********************************************************************************************
565 
566  private:
567  //**********************************************************************************************
569  template< typename MT >
571  static constexpr bool VectorizedAssign_v =
572  ( useOptimizedKernels &&
573  simdEnabled && MT::simdEnabled &&
574  IsSIMDCombinable_v< Type, ElementType_t<MT> > );
576  //**********************************************************************************************
577 
578  //**********************************************************************************************
580  template< typename MT >
582  static constexpr bool VectorizedAddAssign_v =
583  ( useOptimizedKernels &&
584  simdEnabled && MT::simdEnabled &&
585  IsSIMDCombinable_v< Type, ElementType_t<MT> > &&
586  HasSIMDAdd_v< Type, ElementType_t<MT> > &&
587  !IsDiagonal_v<MT> );
589  //**********************************************************************************************
590 
591  //**********************************************************************************************
593  template< typename MT >
595  static constexpr bool VectorizedSubAssign_v =
596  ( useOptimizedKernels &&
597  simdEnabled && MT::simdEnabled &&
598  IsSIMDCombinable_v< Type, ElementType_t<MT> > &&
599  HasSIMDSub_v< Type, ElementType_t<MT> > &&
600  !IsDiagonal_v<MT> );
602  //**********************************************************************************************
603 
604  //**********************************************************************************************
606  template< typename MT >
608  static constexpr bool VectorizedSchurAssign_v =
609  ( useOptimizedKernels &&
610  simdEnabled && MT::simdEnabled &&
611  IsSIMDCombinable_v< Type, ElementType_t<MT> > &&
612  HasSIMDMult_v< Type, ElementType_t<MT> > );
614  //**********************************************************************************************
615 
616  //**SIMD properties*****************************************************************************
618  static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
619  //**********************************************************************************************
620 
621  public:
622  //**Expression template evaluation functions****************************************************
625  template< typename Other > inline bool canAlias ( const Other* alias ) const noexcept;
626  template< typename Other > inline bool isAliased( const Other* alias ) const noexcept;
627 
628  inline bool isAligned () const noexcept;
629  inline bool canSMPAssign() const noexcept;
630 
631  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
632  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
633  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
634 
635  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
636  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
637  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
638  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
639 
640  template< typename MT >
641  inline auto assign( const DenseMatrix<MT,SO>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT> >;
642 
643  template< typename MT >
644  inline auto assign( const DenseMatrix<MT,SO>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT> >;
645 
646  template< typename MT > inline void assign( const DenseMatrix<MT,!SO>& rhs );
647  template< typename MT > inline void assign( const SparseMatrix<MT,SO>& rhs );
648  template< typename MT > inline void assign( const SparseMatrix<MT,!SO>& rhs );
649 
650  template< typename MT >
651  inline auto addAssign( const DenseMatrix<MT,SO>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT> >;
652 
653  template< typename MT >
654  inline auto addAssign( const DenseMatrix<MT,SO>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT> >;
655 
656  template< typename MT > inline void addAssign( const DenseMatrix<MT,!SO>& rhs );
657  template< typename MT > inline void addAssign( const SparseMatrix<MT,SO>& rhs );
658  template< typename MT > inline void addAssign( const SparseMatrix<MT,!SO>& rhs );
659 
660  template< typename MT >
661  inline auto subAssign( const DenseMatrix<MT,SO>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT> >;
662 
663  template< typename MT >
664  inline auto subAssign( const DenseMatrix<MT,SO>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT> >;
665 
666  template< typename MT > inline void subAssign( const DenseMatrix<MT,!SO>& rhs );
667  template< typename MT > inline void subAssign( const SparseMatrix<MT,SO>& rhs );
668  template< typename MT > inline void subAssign( const SparseMatrix<MT,!SO>& rhs );
669 
670  template< typename MT >
671  inline auto schurAssign( const DenseMatrix<MT,SO>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT> >;
672 
673  template< typename MT >
674  inline auto schurAssign( const DenseMatrix<MT,SO>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT> >;
675 
676  template< typename MT > inline void schurAssign( const DenseMatrix<MT,!SO>& rhs );
677  template< typename MT > inline void schurAssign( const SparseMatrix<MT,SO>& rhs );
678  template< typename MT > inline void schurAssign( const SparseMatrix<MT,!SO>& rhs );
680  //**********************************************************************************************
681 
682  private:
683  //**Member variables****************************************************************************
686  size_t m_;
687  size_t n_;
688  size_t nn_;
689  Type* v_;
690 
700  //**********************************************************************************************
701 
702  //**Compile time checks*************************************************************************
708  //**********************************************************************************************
709 };
710 //*************************************************************************************************
711 
712 
713 
714 
715 //=================================================================================================
716 //
717 // CONSTRUCTORS
718 //
719 //=================================================================================================
720 
721 //*************************************************************************************************
724 template< typename Type // Data type of the matrix
725  , bool AF // Alignment flag
726  , bool PF // Padding flag
727  , bool SO // Storage order
728  , typename RT > // Result type
729 inline CustomMatrix<Type,AF,PF,SO,RT>::CustomMatrix()
730  : m_ ( 0UL ) // The current number of rows of the matrix
731  , n_ ( 0UL ) // The current number of columns of the matrix
732  , nn_( 0UL ) // The number of elements between two rows
733  , v_ ( nullptr ) // The custom array of elements
734 {}
735 //*************************************************************************************************
736 
737 
738 //*************************************************************************************************
758 template< typename Type // Data type of the matrix
759  , bool AF // Alignment flag
760  , bool PF // Padding flag
761  , bool SO // Storage order
762  , typename RT > // Result type
763 inline CustomMatrix<Type,AF,PF,SO,RT>::CustomMatrix( Type* ptr, size_t m, size_t n )
764  : m_ ( m ) // The current number of rows of the matrix
765  , n_ ( n ) // The current number of columns of the matrix
766  , nn_( n ) // The number of elements between two rows
767  , v_ ( ptr ) // The custom array of elements
768 {
769  BLAZE_STATIC_ASSERT( PF == unpadded );
770 
771  if( ptr == nullptr ) {
772  BLAZE_THROW_INVALID_ARGUMENT( "Invalid array of elements" );
773  }
774 
775  if( AF && ( !checkAlignment( ptr ) || nn_ % SIMDSIZE != 0UL ) ) {
776  BLAZE_THROW_INVALID_ARGUMENT( "Invalid alignment detected" );
777  }
778 }
779 //*************************************************************************************************
780 
781 
782 //*************************************************************************************************
804 template< typename Type // Data type of the matrix
805  , bool AF // Alignment flag
806  , bool PF // Padding flag
807  , bool SO // Storage order
808  , typename RT > // Result type
809 inline CustomMatrix<Type,AF,PF,SO,RT>::CustomMatrix( Type* ptr, size_t m, size_t n, size_t nn )
810  : m_ ( m ) // The current number of rows of the matrix
811  , n_ ( n ) // The current number of columns of the matrix
812  , nn_( nn ) // The number of elements between two rows
813  , v_ ( ptr ) // The custom array of elements
814 {
815  using blaze::clear;
816 
817  using ClearFunctor = If_t< PF || !IsConst_v<Type>, Clear, Noop >;
818 
819  if( ptr == nullptr ) {
820  BLAZE_THROW_INVALID_ARGUMENT( "Invalid array of elements" );
821  }
822 
823  if( AF && ( !checkAlignment( ptr ) || nn_ % SIMDSIZE != 0UL ) ) {
824  BLAZE_THROW_INVALID_ARGUMENT( "Invalid alignment detected" );
825  }
826 
827  if( PF && IsVectorizable_v<Type> && ( nn_ < nextMultiple<size_t>( n_, SIMDSIZE ) ) ) {
828  BLAZE_THROW_INVALID_ARGUMENT( "Insufficient capacity for padded matrix" );
829  }
830 
831  if( PF && IsVectorizable_v<Type> ) {
832  ClearFunctor clear;
833  for( size_t i=0UL; i<m_; ++i ) {
834  for( size_t j=n_; j<nn_; ++j )
835  clear( v_[i*nn_+j] );
836  }
837  }
838 }
839 //*************************************************************************************************
840 
841 
842 //*************************************************************************************************
849 template< typename Type // Data type of the matrix
850  , bool AF // Alignment flag
851  , bool PF // Padding flag
852  , bool SO // Storage order
853  , typename RT > // Result type
855  : m_ ( m.m_ ) // The current number of rows of the matrix
856  , n_ ( m.n_ ) // The current number of columns of the matrix
857  , nn_( m.nn_ ) // The number of elements between two rows
858  , v_ ( m.v_ ) // The custom array of elements
859 {}
860 //*************************************************************************************************
861 
862 
863 //*************************************************************************************************
868 template< typename Type // Data type of the matrix
869  , bool AF // Alignment flag
870  , bool PF // Padding flag
871  , bool SO // Storage order
872  , typename RT > // Result type
874  : m_ ( m.m_ ) // The current number of rows of the matrix
875  , n_ ( m.n_ ) // The current number of columns of the matrix
876  , nn_( m.nn_ ) // The number of elements between two rows
877  , v_ ( m.v_ ) // The custom array of elements
878 {
879  m.m_ = 0UL;
880  m.n_ = 0UL;
881  m.nn_ = 0UL;
882  m.v_ = nullptr;
883 
884  BLAZE_INTERNAL_ASSERT( m.data() == nullptr, "Invalid data reference detected" );
885 }
886 //*************************************************************************************************
887 
888 
889 
890 
891 //=================================================================================================
892 //
893 // DATA ACCESS FUNCTIONS
894 //
895 //=================================================================================================
896 
897 //*************************************************************************************************
907 template< typename Type // Data type of the matrix
908  , bool AF // Alignment flag
909  , bool PF // Padding flag
910  , bool SO // Storage order
911  , typename RT > // Result type
913  CustomMatrix<Type,AF,PF,SO,RT>::operator()( size_t i, size_t j ) noexcept
914 {
915  BLAZE_USER_ASSERT( i<m_, "Invalid row access index" );
916  BLAZE_USER_ASSERT( j<n_, "Invalid column access index" );
917  return v_[i*nn_+j];
918 }
919 //*************************************************************************************************
920 
921 
922 //*************************************************************************************************
932 template< typename Type // Data type of the matrix
933  , bool AF // Alignment flag
934  , bool PF // Padding flag
935  , bool SO // Storage order
936  , typename RT > // Result type
938  CustomMatrix<Type,AF,PF,SO,RT>::operator()( size_t i, size_t j ) const noexcept
939 {
940  BLAZE_USER_ASSERT( i<m_, "Invalid row access index" );
941  BLAZE_USER_ASSERT( j<n_, "Invalid column access index" );
942  return v_[i*nn_+j];
943 }
944 //*************************************************************************************************
945 
946 
947 //*************************************************************************************************
958 template< typename Type // Data type of the matrix
959  , bool AF // Alignment flag
960  , bool PF // Padding flag
961  , bool SO // Storage order
962  , typename RT > // Result type
965 {
966  if( i >= m_ ) {
967  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
968  }
969  if( j >= n_ ) {
970  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
971  }
972  return (*this)(i,j);
973 }
974 //*************************************************************************************************
975 
976 
977 //*************************************************************************************************
988 template< typename Type // Data type of the matrix
989  , bool AF // Alignment flag
990  , bool PF // Padding flag
991  , bool SO // Storage order
992  , typename RT > // Result type
994  CustomMatrix<Type,AF,PF,SO,RT>::at( size_t i, size_t j ) const
995 {
996  if( i >= m_ ) {
997  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
998  }
999  if( j >= n_ ) {
1000  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
1001  }
1002  return (*this)(i,j);
1003 }
1004 //*************************************************************************************************
1005 
1006 
1007 //*************************************************************************************************
1019 template< typename Type // Data type of the matrix
1020  , bool AF // Alignment flag
1021  , bool PF // Padding flag
1022  , bool SO // Storage order
1023  , typename RT > // Result type
1026 {
1027  return v_;
1028 }
1029 //*************************************************************************************************
1030 
1031 
1032 //*************************************************************************************************
1044 template< typename Type // Data type of the matrix
1045  , bool AF // Alignment flag
1046  , bool PF // Padding flag
1047  , bool SO // Storage order
1048  , typename RT > // Result type
1051 {
1052  return v_;
1053 }
1054 //*************************************************************************************************
1055 
1056 
1057 //*************************************************************************************************
1065 template< typename Type // Data type of the matrix
1066  , bool AF // Alignment flag
1067  , bool PF // Padding flag
1068  , bool SO // Storage order
1069  , typename RT > // Result type
1072 {
1073  BLAZE_USER_ASSERT( i < m_, "Invalid dense matrix row access index" );
1074  return v_+i*nn_;
1075 }
1076 //*************************************************************************************************
1077 
1078 
1079 //*************************************************************************************************
1087 template< typename Type // Data type of the matrix
1088  , bool AF // Alignment flag
1089  , bool PF // Padding flag
1090  , bool SO // Storage order
1091  , typename RT > // Result type
1093  CustomMatrix<Type,AF,PF,SO,RT>::data( size_t i ) const noexcept
1094 {
1095  BLAZE_USER_ASSERT( i < m_, "Invalid dense matrix row access index" );
1096  return v_+i*nn_;
1097 }
1098 //*************************************************************************************************
1099 
1100 
1101 //*************************************************************************************************
1112 template< typename Type // Data type of the matrix
1113  , bool AF // Alignment flag
1114  , bool PF // Padding flag
1115  , bool SO // Storage order
1116  , typename RT > // Result type
1119 {
1120  BLAZE_USER_ASSERT( i < m_, "Invalid dense matrix row access index" );
1121  return Iterator( v_+i*nn_ );
1122 }
1123 //*************************************************************************************************
1124 
1125 
1126 //*************************************************************************************************
1137 template< typename Type // Data type of the matrix
1138  , bool AF // Alignment flag
1139  , bool PF // Padding flag
1140  , bool SO // Storage order
1141  , typename RT > // Result type
1143  CustomMatrix<Type,AF,PF,SO,RT>::begin( size_t i ) const noexcept
1144 {
1145  BLAZE_USER_ASSERT( i < m_, "Invalid dense matrix row access index" );
1146  return ConstIterator( v_+i*nn_ );
1147 }
1148 //*************************************************************************************************
1149 
1150 
1151 //*************************************************************************************************
1162 template< typename Type // Data type of the matrix
1163  , bool AF // Alignment flag
1164  , bool PF // Padding flag
1165  , bool SO // Storage order
1166  , typename RT > // Result type
1168  CustomMatrix<Type,AF,PF,SO,RT>::cbegin( size_t i ) const noexcept
1169 {
1170  BLAZE_USER_ASSERT( i < m_, "Invalid dense matrix row access index" );
1171  return ConstIterator( v_+i*nn_ );
1172 }
1173 //*************************************************************************************************
1174 
1175 
1176 //*************************************************************************************************
1187 template< typename Type // Data type of the matrix
1188  , bool AF // Alignment flag
1189  , bool PF // Padding flag
1190  , bool SO // Storage order
1191  , typename RT > // Result type
1194 {
1195  BLAZE_USER_ASSERT( i < m_, "Invalid dense matrix row access index" );
1196  return Iterator( v_+i*nn_+n_ );
1197 }
1198 //*************************************************************************************************
1199 
1200 
1201 //*************************************************************************************************
1212 template< typename Type // Data type of the matrix
1213  , bool AF // Alignment flag
1214  , bool PF // Padding flag
1215  , bool SO // Storage order
1216  , typename RT > // Result type
1218  CustomMatrix<Type,AF,PF,SO,RT>::end( size_t i ) const noexcept
1219 {
1220  BLAZE_USER_ASSERT( i < m_, "Invalid dense matrix row access index" );
1221  return ConstIterator( v_+i*nn_+n_ );
1222 }
1223 //*************************************************************************************************
1224 
1225 
1226 //*************************************************************************************************
1237 template< typename Type // Data type of the matrix
1238  , bool AF // Alignment flag
1239  , bool PF // Padding flag
1240  , bool SO // Storage order
1241  , typename RT > // Result type
1243  CustomMatrix<Type,AF,PF,SO,RT>::cend( size_t i ) const noexcept
1244 {
1245  BLAZE_USER_ASSERT( i < m_, "Invalid dense matrix row access index" );
1246  return ConstIterator( v_+i*nn_+n_ );
1247 }
1248 //*************************************************************************************************
1249 
1250 
1251 
1252 
1253 //=================================================================================================
1254 //
1255 // ASSIGNMENT OPERATORS
1256 //
1257 //=================================================================================================
1258 
1259 //*************************************************************************************************
1265 template< typename Type // Data type of the matrix
1266  , bool AF // Alignment flag
1267  , bool PF // Padding flag
1268  , bool SO // Storage order
1269  , typename RT > // Result type
1272 {
1273  for( size_t i=0UL; i<m_; ++i )
1274  for( size_t j=0UL; j<n_; ++j )
1275  v_[i*nn_+j] = rhs;
1276 
1277  return *this;
1278 }
1279 //*************************************************************************************************
1280 
1281 
1282 //*************************************************************************************************
1310 template< typename Type // Data type of the matrix
1311  , bool AF // Alignment flag
1312  , bool PF // Padding flag
1313  , bool SO // Storage order
1314  , typename RT > // Result type
1317 {
1318  if( list.size() != m_ || determineColumns( list ) > n_ ) {
1319  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to custom matrix" );
1320  }
1321 
1322  size_t i( 0UL );
1323 
1324  for( const auto& rowList : list ) {
1325  std::fill( std::copy( rowList.begin(), rowList.end(), v_+i*nn_ ),
1326  v_+i*nn_+( PF ? nn_ : n_ ), Type() );
1327  ++i;
1328  }
1329 
1330  return *this;
1331 }
1332 //*************************************************************************************************
1333 
1334 
1335 //*************************************************************************************************
1364 template< typename Type // Data type of the matrix
1365  , bool AF // Alignment flag
1366  , bool PF // Padding flag
1367  , bool SO // Storage order
1368  , typename RT > // Result type
1369 template< typename Other // Data type of the initialization array
1370  , size_t Rows // Number of rows of the initialization array
1371  , size_t Cols > // Number of columns of the initialization array
1373  CustomMatrix<Type,AF,PF,SO,RT>::operator=( const Other (&array)[Rows][Cols] )
1374 {
1375  if( m_ != Rows || n_ != Cols ) {
1376  BLAZE_THROW_INVALID_ARGUMENT( "Invalid array size" );
1377  }
1378 
1379  for( size_t i=0UL; i<Rows; ++i )
1380  for( size_t j=0UL; j<Cols; ++j )
1381  v_[i*nn_+j] = array[i][j];
1382 
1383  return *this;
1384 }
1385 //*************************************************************************************************
1386 
1387 
1388 //*************************************************************************************************
1398 template< typename Type // Data type of the matrix
1399  , bool AF // Alignment flag
1400  , bool PF // Padding flag
1401  , bool SO // Storage order
1402  , typename RT > // Result type
1405 {
1406  if( rhs.rows() != m_ || rhs.columns() != n_ ) {
1407  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1408  }
1409 
1410  smpAssign( *this, ~rhs );
1411 
1412  return *this;
1413 }
1414 //*************************************************************************************************
1415 
1416 
1417 //*************************************************************************************************
1423 template< typename Type // Data type of the matrix
1424  , bool AF // Alignment flag
1425  , bool PF // Padding flag
1426  , bool SO // Storage order
1427  , typename RT > // Result type
1430 {
1431  m_ = rhs.m_;
1432  n_ = rhs.n_;
1433  nn_ = rhs.nn_;
1434  v_ = rhs.v_;
1435 
1436  rhs.m_ = 0UL;
1437  rhs.n_ = 0UL;
1438  rhs.nn_ = 0UL;
1439  rhs.v_ = nullptr;
1440 
1441  BLAZE_INTERNAL_ASSERT( rhs.data() == nullptr, "Invalid data reference detected" );
1442 
1443  return *this;
1444 }
1445 //*************************************************************************************************
1446 
1447 
1448 //*************************************************************************************************
1458 template< typename Type // Data type of the matrix
1459  , bool AF // Alignment flag
1460  , bool PF // Padding flag
1461  , bool SO // Storage order
1462  , typename RT > // Result type
1463 template< typename MT // Type of the right-hand side matrix
1464  , bool SO2 > // Storage order of the right-hand side matrix
1467 {
1468  using TT = decltype( trans( *this ) );
1469  using CT = decltype( ctrans( *this ) );
1470  using IT = decltype( inv( *this ) );
1471 
1472  if( (~rhs).rows() != m_ || (~rhs).columns() != n_ ) {
1473  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1474  }
1475 
1476  if( IsSame_v<MT,TT> && (~rhs).isAliased( this ) ) {
1477  transpose();
1478  }
1479  else if( IsSame_v<MT,CT> && (~rhs).isAliased( this ) ) {
1480  ctranspose();
1481  }
1482  else if( !IsSame_v<MT,IT> && (~rhs).canAlias( this ) ) {
1483  const ResultType_t<MT> tmp( ~rhs );
1484  smpAssign( *this, tmp );
1485  }
1486  else {
1487  if( IsSparseMatrix_v<MT> )
1488  reset();
1489  smpAssign( *this, ~rhs );
1490  }
1491 
1492  return *this;
1493 }
1494 //*************************************************************************************************
1495 
1496 
1497 //*************************************************************************************************
1507 template< typename Type // Data type of the matrix
1508  , bool AF // Alignment flag
1509  , bool PF // Padding flag
1510  , bool SO // Storage order
1511  , typename RT > // Result type
1512 template< typename MT // Type of the right-hand side matrix
1513  , bool SO2 > // Storage order of the right-hand side matrix
1516 {
1517  if( (~rhs).rows() != m_ || (~rhs).columns() != n_ ) {
1518  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1519  }
1520 
1521  if( (~rhs).canAlias( this ) ) {
1522  const ResultType_t<MT> tmp( ~rhs );
1523  smpAddAssign( *this, tmp );
1524  }
1525  else {
1526  smpAddAssign( *this, ~rhs );
1527  }
1528 
1529  return *this;
1530 }
1531 //*************************************************************************************************
1532 
1533 
1534 //*************************************************************************************************
1544 template< typename Type // Data type of the matrix
1545  , bool AF // Alignment flag
1546  , bool PF // Padding flag
1547  , bool SO // Storage order
1548  , typename RT > // Result type
1549 template< typename MT // Type of the right-hand side matrix
1550  , bool SO2 > // Storage order of the right-hand side matrix
1553 {
1554  if( (~rhs).rows() != m_ || (~rhs).columns() != n_ ) {
1555  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1556  }
1557 
1558  if( (~rhs).canAlias( this ) ) {
1559  const ResultType_t<MT> tmp( ~rhs );
1560  smpSubAssign( *this, tmp );
1561  }
1562  else {
1563  smpSubAssign( *this, ~rhs );
1564  }
1565 
1566  return *this;
1567 }
1568 //*************************************************************************************************
1569 
1570 
1571 //*************************************************************************************************
1581 template< typename Type // Data type of the matrix
1582  , bool AF // Alignment flag
1583  , bool PF // Padding flag
1584  , bool SO // Storage order
1585  , typename RT > // Result type
1586 template< typename MT // Type of the right-hand side matrix
1587  , bool SO2 > // Storage order of the right-hand side matrix
1590 {
1591  if( (~rhs).rows() != m_ || (~rhs).columns() != n_ ) {
1592  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1593  }
1594 
1595  if( (~rhs).canAlias( this ) ) {
1596  const ResultType_t<MT> tmp( ~rhs );
1597  smpSchurAssign( *this, tmp );
1598  }
1599  else {
1600  smpSchurAssign( *this, ~rhs );
1601  }
1602 
1603  return *this;
1604 }
1605 //*************************************************************************************************
1606 
1607 
1608 
1609 
1610 //=================================================================================================
1611 //
1612 // UTILITY FUNCTIONS
1613 //
1614 //=================================================================================================
1615 
1616 //*************************************************************************************************
1621 template< typename Type // Data type of the matrix
1622  , bool AF // Alignment flag
1623  , bool PF // Padding flag
1624  , bool SO // Storage order
1625  , typename RT > // Result type
1626 inline size_t CustomMatrix<Type,AF,PF,SO,RT>::rows() const noexcept
1627 {
1628  return m_;
1629 }
1630 //*************************************************************************************************
1631 
1632 
1633 //*************************************************************************************************
1638 template< typename Type // Data type of the matrix
1639  , bool AF // Alignment flag
1640  , bool PF // Padding flag
1641  , bool SO // Storage order
1642  , typename RT > // Result type
1643 inline size_t CustomMatrix<Type,AF,PF,SO,RT>::columns() const noexcept
1644 {
1645  return n_;
1646 }
1647 //*************************************************************************************************
1648 
1649 
1650 //*************************************************************************************************
1660 template< typename Type // Data type of the matrix
1661  , bool AF // Alignment flag
1662  , bool PF // Padding flag
1663  , bool SO // Storage order
1664  , typename RT > // Result type
1665 inline size_t CustomMatrix<Type,AF,PF,SO,RT>::spacing() const noexcept
1666 {
1667  return nn_;
1668 }
1669 //*************************************************************************************************
1670 
1671 
1672 //*************************************************************************************************
1677 template< typename Type // Data type of the matrix
1678  , bool AF // Alignment flag
1679  , bool PF // Padding flag
1680  , bool SO // Storage order
1681  , typename RT > // Result type
1682 inline size_t CustomMatrix<Type,AF,PF,SO,RT>::capacity() const noexcept
1683 {
1684  return m_ * nn_;
1685 }
1686 //*************************************************************************************************
1687 
1688 
1689 //*************************************************************************************************
1700 template< typename Type // Data type of the matrix
1701  , bool AF // Alignment flag
1702  , bool PF // Padding flag
1703  , bool SO // Storage order
1704  , typename RT > // Result type
1705 inline size_t CustomMatrix<Type,AF,PF,SO,RT>::capacity( size_t i ) const noexcept
1706 {
1707  MAYBE_UNUSED( i );
1708  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1709  return nn_;
1710 }
1711 //*************************************************************************************************
1712 
1713 
1714 //*************************************************************************************************
1719 template< typename Type // Data type of the matrix
1720  , bool AF // Alignment flag
1721  , bool PF // Padding flag
1722  , bool SO // Storage order
1723  , typename RT > // Result type
1725 {
1726  size_t nonzeros( 0UL );
1727 
1728  for( size_t i=0UL; i<m_; ++i )
1729  for( size_t j=0UL; j<n_; ++j )
1730  if( !isDefault( v_[i*nn_+j] ) )
1731  ++nonzeros;
1732 
1733  return nonzeros;
1734 }
1735 //*************************************************************************************************
1736 
1737 
1738 //*************************************************************************************************
1749 template< typename Type // Data type of the matrix
1750  , bool AF // Alignment flag
1751  , bool PF // Padding flag
1752  , bool SO // Storage order
1753  , typename RT > // Result type
1754 inline size_t CustomMatrix<Type,AF,PF,SO,RT>::nonZeros( size_t i ) const
1755 {
1756  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1757 
1758  const size_t jend( i*nn_ + n_ );
1759  size_t nonzeros( 0UL );
1760 
1761  for( size_t j=i*nn_; j<jend; ++j )
1762  if( !isDefault( v_[j] ) )
1763  ++nonzeros;
1764 
1765  return nonzeros;
1766 }
1767 //*************************************************************************************************
1768 
1769 
1770 //*************************************************************************************************
1775 template< typename Type // Data type of the matrix
1776  , bool AF // Alignment flag
1777  , bool PF // Padding flag
1778  , bool SO // Storage order
1779  , typename RT > // Result type
1781 {
1782  using blaze::clear;
1783 
1784  for( size_t i=0UL; i<m_; ++i )
1785  for( size_t j=0UL; j<n_; ++j )
1786  clear( v_[i*nn_+j] );
1787 }
1788 //*************************************************************************************************
1789 
1790 
1791 //*************************************************************************************************
1802 template< typename Type // Data type of the matrix
1803  , bool AF // Alignment flag
1804  , bool PF // Padding flag
1805  , bool SO // Storage order
1806  , typename RT > // Result type
1808 {
1809  using blaze::clear;
1810 
1811  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1812  for( size_t j=0UL; j<n_; ++j )
1813  clear( v_[i*nn_+j] );
1814 }
1815 //*************************************************************************************************
1816 
1817 
1818 //*************************************************************************************************
1825 template< typename Type // Data type of the matrix
1826  , bool AF // Alignment flag
1827  , bool PF // Padding flag
1828  , bool SO // Storage order
1829  , typename RT > // Result type
1831 {
1832  m_ = 0UL;
1833  n_ = 0UL;
1834  nn_ = 0UL;
1835  v_ = nullptr;
1836 }
1837 //*************************************************************************************************
1838 
1839 
1840 //*************************************************************************************************
1846 template< typename Type // Data type of the matrix
1847  , bool AF // Alignment flag
1848  , bool PF // Padding flag
1849  , bool SO // Storage order
1850  , typename RT > // Result type
1852 {
1853  using std::swap;
1854 
1855  swap( m_ , m.m_ );
1856  swap( n_ , m.n_ );
1857  swap( nn_, m.nn_ );
1858  swap( v_ , m.v_ );
1859 }
1860 //*************************************************************************************************
1861 
1862 
1863 
1864 
1865 //=================================================================================================
1866 //
1867 // NUMERIC FUNCTIONS
1868 //
1869 //=================================================================================================
1870 
1871 //*************************************************************************************************
1879 template< typename Type // Data type of the matrix
1880  , bool AF // Alignment flag
1881  , bool PF // Padding flag
1882  , bool SO // Storage order
1883  , typename RT > // Result type
1885 {
1886  using std::swap;
1887 
1888  if( m_ != n_ ) {
1889  BLAZE_THROW_LOGIC_ERROR( "Impossible transpose operation" );
1890  }
1891 
1892  for( size_t i=1UL; i<m_; ++i )
1893  for( size_t j=0UL; j<i; ++j )
1894  swap( v_[i*nn_+j], v_[j*nn_+i] );
1895 
1896  return *this;
1897 }
1898 //*************************************************************************************************
1899 
1900 
1901 //*************************************************************************************************
1909 template< typename Type // Data type of the matrix
1910  , bool AF // Alignment flag
1911  , bool PF // Padding flag
1912  , bool SO // Storage order
1913  , typename RT > // Result type
1915 {
1916  if( m_ != n_ ) {
1917  BLAZE_THROW_LOGIC_ERROR( "Impossible transpose operation" );
1918  }
1919 
1920  for( size_t i=0UL; i<m_; ++i ) {
1921  for( size_t j=0UL; j<i; ++j ) {
1922  cswap( v_[i*nn_+j], v_[j*nn_+i] );
1923  }
1924  conjugate( v_[i*nn_+i] );
1925  }
1926 
1927  return *this;
1928 }
1929 //*************************************************************************************************
1930 
1931 
1932 //*************************************************************************************************
1953 template< typename Type // Data type of the matrix
1954  , bool AF // Alignment flag
1955  , bool PF // Padding flag
1956  , bool SO // Storage order
1957  , typename RT > // Result type
1958 template< typename Other > // Data type of the scalar value
1960 {
1961  for( size_t i=0UL; i<m_; ++i )
1962  for( size_t j=0UL; j<n_; ++j )
1963  v_[i*nn_+j] *= scalar;
1964 
1965  return *this;
1966 }
1967 //*************************************************************************************************
1968 
1969 
1970 
1971 
1972 //=================================================================================================
1973 //
1974 // RESOURCE MANAGEMENT FUNCTIONS
1975 //
1976 //=================================================================================================
1977 
1978 //*************************************************************************************************
2001 template< typename Type // Data type of the matrix
2002  , bool AF // Alignment flag
2003  , bool PF // Padding flag
2004  , bool SO // Storage order
2005  , typename RT > // Result type
2006 inline void CustomMatrix<Type,AF,PF,SO,RT>::reset( Type* ptr, size_t m, size_t n )
2007 {
2008  BLAZE_STATIC_ASSERT( PF == unpadded );
2009 
2010  CustomMatrix tmp( ptr, m, n );
2011  swap( tmp );
2012 }
2013 //*************************************************************************************************
2014 
2015 
2016 //*************************************************************************************************
2039 template< typename Type // Data type of the matrix
2040  , bool AF // Alignment flag
2041  , bool PF // Padding flag
2042  , bool SO // Storage order
2043  , typename RT > // Result type
2044 inline void CustomMatrix<Type,AF,PF,SO,RT>::reset( Type* ptr, size_t m, size_t n, size_t nn )
2045 {
2046  CustomMatrix tmp( ptr, m, n, nn );
2047  swap( tmp );
2048 }
2049 //*************************************************************************************************
2050 
2051 
2052 
2053 
2054 //=================================================================================================
2055 //
2056 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
2057 //
2058 //=================================================================================================
2059 
2060 //*************************************************************************************************
2070 template< typename Type // Data type of the matrix
2071  , bool AF // Alignment flag
2072  , bool PF // Padding flag
2073  , bool SO // Storage order
2074  , typename RT > // Result type
2075 template< typename Other > // Data type of the foreign expression
2076 inline bool CustomMatrix<Type,AF,PF,SO,RT>::canAlias( const Other* alias ) const noexcept
2077 {
2078  return static_cast<const void*>( this ) == static_cast<const void*>( alias );
2079 }
2080 //*************************************************************************************************
2081 
2082 
2083 //*************************************************************************************************
2093 template< typename Type // Data type of the matrix
2094  , bool AF // Alignment flag
2095  , bool PF // Padding flag
2096  , bool SO // Storage order
2097  , typename RT > // Result type
2098 template< typename Other > // Data type of the foreign expression
2099 inline bool CustomMatrix<Type,AF,PF,SO,RT>::isAliased( const Other* alias ) const noexcept
2100 {
2101  return static_cast<const void*>( this ) == static_cast<const void*>( alias );
2102 }
2103 //*************************************************************************************************
2104 
2105 
2106 //*************************************************************************************************
2115 template< typename Type // Data type of the matrix
2116  , bool AF // Alignment flag
2117  , bool PF // Padding flag
2118  , bool SO // Storage order
2119  , typename RT > // Result type
2120 inline bool CustomMatrix<Type,AF,PF,SO,RT>::isAligned() const noexcept
2121 {
2122  return ( AF || ( checkAlignment( v_ ) && columns() % SIMDSIZE == 0UL ) );
2123 }
2124 //*************************************************************************************************
2125 
2126 
2127 //*************************************************************************************************
2137 template< typename Type // Data type of the matrix
2138  , bool AF // Alignment flag
2139  , bool PF // Padding flag
2140  , bool SO // Storage order
2141  , typename RT > // Result type
2143 {
2144  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
2145 }
2146 //*************************************************************************************************
2147 
2148 
2149 //*************************************************************************************************
2164 template< typename Type // Data type of the matrix
2165  , bool AF // Alignment flag
2166  , bool PF // Padding flag
2167  , bool SO // Storage order
2168  , typename RT > // Result type
2170  CustomMatrix<Type,AF,PF,SO,RT>::load( size_t i, size_t j ) const noexcept
2171 {
2172  if( AF && PF )
2173  return loada( i, j );
2174  else
2175  return loadu( i, j );
2176 }
2177 //*************************************************************************************************
2178 
2179 
2180 //*************************************************************************************************
2195 template< typename Type // Data type of the matrix
2196  , bool AF // Alignment flag
2197  , bool PF // Padding flag
2198  , bool SO // Storage order
2199  , typename RT > // Result type
2201  CustomMatrix<Type,AF,PF,SO,RT>::loada( size_t i, size_t j ) const noexcept
2202 {
2203  using blaze::loada;
2204 
2206 
2207  BLAZE_INTERNAL_ASSERT( i < m_, "Invalid row access index" );
2208  BLAZE_INTERNAL_ASSERT( j < n_, "Invalid column access index" );
2209  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= ( PF ? nn_ : n_ ), "Invalid column access index" );
2210  BLAZE_INTERNAL_ASSERT( !PF || j % SIMDSIZE == 0UL, "Invalid column access index" );
2211  BLAZE_INTERNAL_ASSERT( checkAlignment( v_+i*nn_+j ), "Invalid alignment detected" );
2212 
2213  return loada( v_+i*nn_+j );
2214 }
2215 //*************************************************************************************************
2216 
2217 
2218 //*************************************************************************************************
2233 template< typename Type // Data type of the matrix
2234  , bool AF // Alignment flag
2235  , bool PF // Padding flag
2236  , bool SO // Storage order
2237  , typename RT > // Result type
2239  CustomMatrix<Type,AF,PF,SO,RT>::loadu( size_t i, size_t j ) const noexcept
2240 {
2241  using blaze::loadu;
2242 
2244 
2245  BLAZE_INTERNAL_ASSERT( i < m_, "Invalid row access index" );
2246  BLAZE_INTERNAL_ASSERT( j < n_, "Invalid column access index" );
2247  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= ( PF ? nn_ : n_ ), "Invalid column access index" );
2248 
2249  return loadu( v_+i*nn_+j );
2250 }
2251 //*************************************************************************************************
2252 
2253 
2254 //*************************************************************************************************
2270 template< typename Type // Data type of the matrix
2271  , bool AF // Alignment flag
2272  , bool PF // Padding flag
2273  , bool SO // Storage order
2274  , typename RT > // Result type
2276  CustomMatrix<Type,AF,PF,SO,RT>::store( size_t i, size_t j, const SIMDType& value ) noexcept
2277 {
2278  if( AF && PF )
2279  storea( i, j, value );
2280  else
2281  storeu( i, j, value );
2282 }
2283 //*************************************************************************************************
2284 
2285 
2286 //*************************************************************************************************
2302 template< typename Type // Data type of the matrix
2303  , bool AF // Alignment flag
2304  , bool PF // Padding flag
2305  , bool SO // Storage order
2306  , typename RT > // Result type
2308  CustomMatrix<Type,AF,PF,SO,RT>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
2309 {
2310  using blaze::storea;
2311 
2313 
2314  BLAZE_INTERNAL_ASSERT( i < m_, "Invalid row access index" );
2315  BLAZE_INTERNAL_ASSERT( j < n_, "Invalid column access index" );
2316  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= ( PF ? nn_ : n_ ), "Invalid column access index" );
2317  BLAZE_INTERNAL_ASSERT( !PF || j % SIMDSIZE == 0UL, "Invalid column access index" );
2318  BLAZE_INTERNAL_ASSERT( checkAlignment( v_+i*nn_+j ), "Invalid alignment detected" );
2319 
2320  storea( v_+i*nn_+j, value );
2321 }
2322 //*************************************************************************************************
2323 
2324 
2325 //*************************************************************************************************
2341 template< typename Type // Data type of the matrix
2342  , bool AF // Alignment flag
2343  , bool PF // Padding flag
2344  , bool SO // Storage order
2345  , typename RT > // Result type
2347  CustomMatrix<Type,AF,PF,SO,RT>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
2348 {
2349  using blaze::storeu;
2350 
2352 
2353  BLAZE_INTERNAL_ASSERT( i < m_, "Invalid row access index" );
2354  BLAZE_INTERNAL_ASSERT( j < n_, "Invalid column access index" );
2355  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= ( PF ? nn_ : n_ ), "Invalid column access index" );
2356 
2357  storeu( v_+i*nn_+j, value );
2358 }
2359 //*************************************************************************************************
2360 
2361 
2362 //*************************************************************************************************
2379 template< typename Type // Data type of the matrix
2380  , bool AF // Alignment flag
2381  , bool PF // Padding flag
2382  , bool SO // Storage order
2383  , typename RT > // Result type
2385  CustomMatrix<Type,AF,PF,SO,RT>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
2386 {
2387  using blaze::stream;
2388 
2390 
2391  BLAZE_INTERNAL_ASSERT( i < m_, "Invalid row access index" );
2392  BLAZE_INTERNAL_ASSERT( j < n_, "Invalid column access index" );
2393  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= ( PF ? nn_ : n_ ), "Invalid column access index" );
2394  BLAZE_INTERNAL_ASSERT( !PF || j % SIMDSIZE == 0UL, "Invalid column access index" );
2395  BLAZE_INTERNAL_ASSERT( checkAlignment( v_+i*nn_+j ), "Invalid alignment detected" );
2396 
2397  stream( v_+i*nn_+j, value );
2398 }
2399 //*************************************************************************************************
2400 
2401 
2402 //*************************************************************************************************
2413 template< typename Type // Data type of the matrix
2414  , bool AF // Alignment flag
2415  , bool PF // Padding flag
2416  , bool SO // Storage order
2417  , typename RT > // Result type
2418 template< typename MT > // Type of the right-hand side dense matrix
2421 {
2422  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2423  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2424 
2425  const size_t jpos( n_ & size_t(-2) );
2426  BLAZE_INTERNAL_ASSERT( ( n_ - ( n_ % 2UL ) ) == jpos, "Invalid end calculation" );
2427 
2428  for( size_t i=0UL; i<m_; ++i ) {
2429  for( size_t j=0UL; j<jpos; j+=2UL ) {
2430  v_[i*nn_+j ] = (~rhs)(i,j );
2431  v_[i*nn_+j+1UL] = (~rhs)(i,j+1UL);
2432  }
2433  if( jpos < n_ ) {
2434  v_[i*nn_+jpos] = (~rhs)(i,jpos);
2435  }
2436  }
2437 }
2438 //*************************************************************************************************
2439 
2440 
2441 //*************************************************************************************************
2452 template< typename Type // Data type of the matrix
2453  , bool AF // Alignment flag
2454  , bool PF // Padding flag
2455  , bool SO // Storage order
2456  , typename RT > // Result type
2457 template< typename MT > // Type of the right-hand side dense matrix
2460 {
2462 
2463  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2464  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2465 
2466  constexpr bool remainder( !PF || !IsPadded_v<MT> );
2467 
2468  const size_t jpos( ( remainder )?( n_ & size_t(-SIMDSIZE) ):( n_ ) );
2469  BLAZE_INTERNAL_ASSERT( !remainder || ( n_ - ( n_ % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
2470 
2471  if( AF && PF && useStreaming &&
2472  ( m_*n_ > ( cacheSize / ( sizeof(Type) * 3UL ) ) ) && !(~rhs).isAliased( this ) )
2473  {
2474  for( size_t i=0UL; i<m_; ++i )
2475  {
2476  size_t j( 0UL );
2477  Iterator left( begin(i) );
2478  ConstIterator_t<MT> right( (~rhs).begin(i) );
2479 
2480  for( ; j<jpos; j+=SIMDSIZE ) {
2481  left.stream( right.load() ); left += SIMDSIZE, right += SIMDSIZE;
2482  }
2483  for( ; remainder && j<n_; ++j ) {
2484  *left = *right; ++left; ++right;
2485  }
2486  }
2487  }
2488  else
2489  {
2490  for( size_t i=0UL; i<m_; ++i )
2491  {
2492  size_t j( 0UL );
2493  Iterator left( begin(i) );
2494  ConstIterator_t<MT> right( (~rhs).begin(i) );
2495 
2496  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
2497  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2498  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2499  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2500  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2501  }
2502  for( ; j<jpos; j+=SIMDSIZE ) {
2503  left.store( right.load() ); left+=SIMDSIZE, right+=SIMDSIZE;
2504  }
2505  for( ; remainder && j<n_; ++j ) {
2506  *left = *right; ++left; ++right;
2507  }
2508  }
2509  }
2510 }
2511 //*************************************************************************************************
2512 
2513 
2514 //*************************************************************************************************
2525 template< typename Type // Data type of the matrix
2526  , bool AF // Alignment flag
2527  , bool PF // Padding flag
2528  , bool SO // Storage order
2529  , typename RT > // Result type
2530 template< typename MT > // Type of the right-hand side dense matrix
2532 {
2534 
2535  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2536  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2537 
2538  constexpr size_t block( BLOCK_SIZE );
2539 
2540  for( size_t ii=0UL; ii<m_; ii+=block ) {
2541  const size_t iend( min( m_, ii+block ) );
2542  for( size_t jj=0UL; jj<n_; jj+=block ) {
2543  const size_t jend( min( n_, jj+block ) );
2544  for( size_t i=ii; i<iend; ++i ) {
2545  for( size_t j=jj; j<jend; ++j ) {
2546  v_[i*nn_+j] = (~rhs)(i,j);
2547  }
2548  }
2549  }
2550  }
2551 }
2552 //*************************************************************************************************
2553 
2554 
2555 //*************************************************************************************************
2566 template< typename Type // Data type of the matrix
2567  , bool AF // Alignment flag
2568  , bool PF // Padding flag
2569  , bool SO // Storage order
2570  , typename RT > // Result type
2571 template< typename MT > // Type of the right-hand side sparse matrix
2573 {
2574  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2575  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2576 
2577  for( size_t i=0UL; i<m_; ++i )
2578  for( auto element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
2579  v_[i*nn_+element->index()] = element->value();
2580 }
2581 //*************************************************************************************************
2582 
2583 
2584 //*************************************************************************************************
2595 template< typename Type // Data type of the matrix
2596  , bool AF // Alignment flag
2597  , bool PF // Padding flag
2598  , bool SO // Storage order
2599  , typename RT > // Result type
2600 template< typename MT > // Type of the right-hand side sparse matrix
2602 {
2604 
2605  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2606  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2607 
2608  for( size_t j=0UL; j<n_; ++j )
2609  for( auto element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
2610  v_[element->index()*nn_+j] = element->value();
2611 }
2612 //*************************************************************************************************
2613 
2614 
2615 //*************************************************************************************************
2626 template< typename Type // Data type of the matrix
2627  , bool AF // Alignment flag
2628  , bool PF // Padding flag
2629  , bool SO // Storage order
2630  , typename RT > // Result type
2631 template< typename MT > // Type of the right-hand side dense matrix
2634 {
2635  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2636  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2637 
2638  for( size_t i=0UL; i<m_; ++i )
2639  {
2640  if( IsDiagonal_v<MT> )
2641  {
2642  v_[i*nn_+i] += (~rhs)(i,i);
2643  }
2644  else
2645  {
2646  const size_t jbegin( ( IsUpper_v<MT> )
2647  ?( IsStrictlyUpper_v<MT> ? i+1UL : i )
2648  :( 0UL ) );
2649  const size_t jend ( ( IsLower_v<MT> )
2650  ?( IsStrictlyLower_v<MT> ? i : i+1UL )
2651  :( n_ ) );
2652  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
2653 
2654  size_t j( jbegin );
2655 
2656  for( ; (j+2UL) <= jend; j+=2UL ) {
2657  v_[i*nn_+j ] += (~rhs)(i,j );
2658  v_[i*nn_+j+1UL] += (~rhs)(i,j+1UL);
2659  }
2660  if( j < jend ) {
2661  v_[i*nn_+j] += (~rhs)(i,j);
2662  }
2663  }
2664  }
2665 }
2666 //*************************************************************************************************
2667 
2668 
2669 //*************************************************************************************************
2680 template< typename Type // Data type of the matrix
2681  , bool AF // Alignment flag
2682  , bool PF // Padding flag
2683  , bool SO // Storage order
2684  , typename RT > // Result type
2685 template< typename MT > // Type of the right-hand side dense matrix
2688 {
2691 
2692  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2693  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2694 
2695  constexpr bool remainder( !PF || !IsPadded_v<MT> );
2696 
2697  for( size_t i=0UL; i<m_; ++i )
2698  {
2699  const size_t jbegin( ( IsUpper_v<MT> )
2700  ?( ( IsStrictlyUpper_v<MT> ? i+1UL : i ) & size_t(-SIMDSIZE) )
2701  :( 0UL ) );
2702  const size_t jend ( ( IsLower_v<MT> )
2703  ?( IsStrictlyLower_v<MT> ? i : i+1UL )
2704  :( n_ ) );
2705  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
2706 
2707  const size_t jpos( ( remainder )?( jend & size_t(-SIMDSIZE) ):( jend ) );
2708  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
2709 
2710  size_t j( jbegin );
2711  Iterator left( begin(i) + jbegin );
2712  ConstIterator_t<MT> right( (~rhs).begin(i) + jbegin );
2713 
2714  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
2715  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2716  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2717  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2718  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2719  }
2720  for( ; j<jpos; j+=SIMDSIZE ) {
2721  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2722  }
2723  for( ; remainder && j<jend; ++j ) {
2724  *left += *right; ++left; ++right;
2725  }
2726  }
2727 }
2728 //*************************************************************************************************
2729 
2730 
2731 //*************************************************************************************************
2742 template< typename Type // Data type of the matrix
2743  , bool AF // Alignment flag
2744  , bool PF // Padding flag
2745  , bool SO // Storage order
2746  , typename RT > // Result type
2747 template< typename MT > // Type of the right-hand side dense matrix
2749 {
2751 
2752  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2753  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2754 
2755  constexpr size_t block( BLOCK_SIZE );
2756 
2757  for( size_t ii=0UL; ii<m_; ii+=block ) {
2758  const size_t iend( min( m_, ii+block ) );
2759  for( size_t jj=0UL; jj<n_; jj+=block )
2760  {
2761  if( IsLower_v<MT> && ii < jj ) break;
2762  if( IsUpper_v<MT> && ii > jj ) continue;
2763 
2764  for( size_t i=ii; i<iend; ++i )
2765  {
2766  const size_t jbegin( ( IsUpper_v<MT> )
2767  ?( max( ( IsStrictlyUpper_v<MT> ? i+1UL : i ), jj ) )
2768  :( jj ) );
2769  const size_t jend ( ( IsLower_v<MT> )
2770  ?( min( ( IsStrictlyLower_v<MT> ? i : i+1UL ), n_, jj+block ) )
2771  :( min( n_, jj+block ) ) );
2772  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
2773 
2774  for( size_t j=jbegin; j<jend; ++j ) {
2775  v_[i*nn_+j] += (~rhs)(i,j);
2776  }
2777  }
2778  }
2779  }
2780 }
2781 //*************************************************************************************************
2782 
2783 
2784 //*************************************************************************************************
2795 template< typename Type // Data type of the matrix
2796  , bool AF // Alignment flag
2797  , bool PF // Padding flag
2798  , bool SO // Storage order
2799  , typename RT > // Result type
2800 template< typename MT > // Type of the right-hand side sparse matrix
2802 {
2803  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2804  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2805 
2806  for( size_t i=0UL; i<m_; ++i )
2807  for( auto element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
2808  v_[i*nn_+element->index()] += element->value();
2809 }
2810 //*************************************************************************************************
2811 
2812 
2813 //*************************************************************************************************
2824 template< typename Type // Data type of the matrix
2825  , bool AF // Alignment flag
2826  , bool PF // Padding flag
2827  , bool SO // Storage order
2828  , typename RT > // Result type
2829 template< typename MT > // Type of the right-hand side sparse matrix
2831 {
2833 
2834  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2835  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2836 
2837  for( size_t j=0UL; j<n_; ++j )
2838  for( auto element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
2839  v_[element->index()*nn_+j] += element->value();
2840 }
2841 //*************************************************************************************************
2842 
2843 
2844 //*************************************************************************************************
2855 template< typename Type // Data type of the matrix
2856  , bool AF // Alignment flag
2857  , bool PF // Padding flag
2858  , bool SO // Storage order
2859  , typename RT > // Result type
2860 template< typename MT > // Type of the right-hand side dense matrix
2863 {
2864  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2865  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2866 
2867  for( size_t i=0UL; i<m_; ++i )
2868  {
2869  if( IsDiagonal_v<MT> )
2870  {
2871  v_[i*nn_+i] -= (~rhs)(i,i);
2872  }
2873  else
2874  {
2875  const size_t jbegin( ( IsUpper_v<MT> )
2876  ?( IsStrictlyUpper_v<MT> ? i+1UL : i )
2877  :( 0UL ) );
2878  const size_t jend ( ( IsLower_v<MT> )
2879  ?( IsStrictlyLower_v<MT> ? i : i+1UL )
2880  :( n_ ) );
2881  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
2882 
2883  size_t j( jbegin );
2884 
2885  for( ; (j+2UL) <= jend; j+=2UL ) {
2886  v_[i*nn_+j ] -= (~rhs)(i,j );
2887  v_[i*nn_+j+1UL] -= (~rhs)(i,j+1UL);
2888  }
2889  if( j < jend ) {
2890  v_[i*nn_+j] -= (~rhs)(i,j);
2891  }
2892  }
2893  }
2894 }
2895 //*************************************************************************************************
2896 
2897 
2898 //*************************************************************************************************
2909 template< typename Type // Data type of the matrix
2910  , bool AF // Alignment flag
2911  , bool PF // Padding flag
2912  , bool SO // Storage order
2913  , typename RT > // Result type
2914 template< typename MT > // Type of the right-hand side dense matrix
2917 {
2920 
2921  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2922  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2923 
2924  constexpr bool remainder( !PF || !IsPadded_v<MT> );
2925 
2926  for( size_t i=0UL; i<m_; ++i )
2927  {
2928  const size_t jbegin( ( IsUpper_v<MT> )
2929  ?( ( IsStrictlyUpper_v<MT> ? i+1UL : i ) & size_t(-SIMDSIZE) )
2930  :( 0UL ) );
2931  const size_t jend ( ( IsLower_v<MT> )
2932  ?( IsStrictlyLower_v<MT> ? i : i+1UL )
2933  :( n_ ) );
2934  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
2935 
2936  const size_t jpos( ( remainder )?( jend & size_t(-SIMDSIZE) ):( jend ) );
2937  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
2938 
2939  size_t j( jbegin );
2940  Iterator left( begin(i) + jbegin );
2941  ConstIterator_t<MT> right( (~rhs).begin(i) + jbegin );
2942 
2943  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
2944  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2945  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2946  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2947  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2948  }
2949  for( ; j<jpos; j+=SIMDSIZE ) {
2950  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2951  }
2952  for( ; remainder && j<jend; ++j ) {
2953  *left -= *right; ++left; ++right;
2954  }
2955  }
2956 }
2957 //*************************************************************************************************
2958 
2959 
2960 //*************************************************************************************************
2971 template< typename Type // Data type of the matrix
2972  , bool AF // Alignment flag
2973  , bool PF // Padding flag
2974  , bool SO // Storage order
2975  , typename RT > // Result type
2976 template< typename MT > // Type of the right-hand side dense matrix
2978 {
2980 
2981  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
2982  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
2983 
2984  constexpr size_t block( BLOCK_SIZE );
2985 
2986  for( size_t ii=0UL; ii<m_; ii+=block ) {
2987  const size_t iend( min( m_, ii+block ) );
2988  for( size_t jj=0UL; jj<n_; jj+=block )
2989  {
2990  if( IsLower_v<MT> && ii < jj ) break;
2991  if( IsUpper_v<MT> && ii > jj ) continue;
2992 
2993  for( size_t i=ii; i<iend; ++i )
2994  {
2995  const size_t jbegin( ( IsUpper_v<MT> )
2996  ?( max( ( IsStrictlyUpper_v<MT> ? i+1UL : i ), jj ) )
2997  :( jj ) );
2998  const size_t jend ( ( IsLower_v<MT> )
2999  ?( min( ( IsStrictlyLower_v<MT> ? i : i+1UL ), n_, jj+block ) )
3000  :( min( n_, jj+block ) ) );
3001  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
3002 
3003  for( size_t j=jbegin; j<jend; ++j ) {
3004  v_[i*nn_+j] -= (~rhs)(i,j);
3005  }
3006  }
3007  }
3008  }
3009 }
3010 //*************************************************************************************************
3011 
3012 
3013 //*************************************************************************************************
3024 template< typename Type // Data type of the matrix
3025  , bool AF // Alignment flag
3026  , bool PF // Padding flag
3027  , bool SO // Storage order
3028  , typename RT > // Result type
3029 template< typename MT > // Type of the right-hand side sparse matrix
3031 {
3032  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
3033  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
3034 
3035  for( size_t i=0UL; i<m_; ++i )
3036  for( auto element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
3037  v_[i*nn_+element->index()] -= element->value();
3038 }
3039 //*************************************************************************************************
3040 
3041 
3042 //*************************************************************************************************
3053 template< typename Type // Data type of the matrix
3054  , bool AF // Alignment flag
3055  , bool PF // Padding flag
3056  , bool SO // Storage order
3057  , typename RT > // Result type
3058 template< typename MT > // Type of the right-hand side sparse matrix
3060 {
3062 
3063  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
3064  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
3065 
3066  for( size_t j=0UL; j<n_; ++j )
3067  for( auto element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
3068  v_[element->index()*nn_+j] -= element->value();
3069 }
3070 //*************************************************************************************************
3071 
3072 
3073 //*************************************************************************************************
3084 template< typename Type // Data type of the matrix
3085  , bool AF // Alignment flag
3086  , bool PF // Padding flag
3087  , bool SO // Storage order
3088  , typename RT > // Result type
3089 template< typename MT > // Type of the right-hand side dense matrix
3092 {
3093  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
3094  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
3095 
3096  const size_t jpos( n_ & size_t(-2) );
3097  BLAZE_INTERNAL_ASSERT( ( n_ - ( n_ % 2UL ) ) == jpos, "Invalid end calculation" );
3098 
3099  for( size_t i=0UL; i<m_; ++i ) {
3100  for( size_t j=0UL; j<jpos; j+=2UL ) {
3101  v_[i*nn_+j ] *= (~rhs)(i,j );
3102  v_[i*nn_+j+1UL] *= (~rhs)(i,j+1UL);
3103  }
3104  if( jpos < n_ ) {
3105  v_[i*nn_+jpos] *= (~rhs)(i,jpos);
3106  }
3107  }
3108 }
3109 //*************************************************************************************************
3110 
3111 
3112 //*************************************************************************************************
3123 template< typename Type // Data type of the matrix
3124  , bool AF // Alignment flag
3125  , bool PF // Padding flag
3126  , bool SO // Storage order
3127  , typename RT > // Result type
3128 template< typename MT > // Type of the right-hand side dense matrix
3131 {
3133 
3134  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
3135  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
3136 
3137  constexpr bool remainder( !PF || !IsPadded_v<MT> );
3138 
3139  for( size_t i=0UL; i<m_; ++i )
3140  {
3141  const size_t jpos( ( remainder )?( n_ & size_t(-SIMDSIZE) ):( n_ ) );
3142  BLAZE_INTERNAL_ASSERT( !remainder || ( n_ - ( n_ % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
3143 
3144  size_t j( 0UL );
3145  Iterator left( begin(i) );
3146  ConstIterator_t<MT> right( (~rhs).begin(i) );
3147 
3148  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
3149  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3150  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3151  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3152  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3153  }
3154  for( ; j<jpos; j+=SIMDSIZE ) {
3155  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3156  }
3157  for( ; remainder && j<n_; ++j ) {
3158  *left *= *right; ++left; ++right;
3159  }
3160  }
3161 }
3162 //*************************************************************************************************
3163 
3164 
3165 //*************************************************************************************************
3176 template< typename Type // Data type of the matrix
3177  , bool AF // Alignment flag
3178  , bool PF // Padding flag
3179  , bool SO // Storage order
3180  , typename RT > // Result type
3181 template< typename MT > // Type of the right-hand side dense matrix
3183 {
3185 
3186  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
3187  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
3188 
3189  constexpr size_t block( BLOCK_SIZE );
3190 
3191  for( size_t ii=0UL; ii<m_; ii+=block ) {
3192  const size_t iend( min( m_, ii+block ) );
3193  for( size_t jj=0UL; jj<n_; jj+=block ) {
3194  const size_t jend( min( n_, jj+block ) );
3195  for( size_t i=ii; i<iend; ++i ) {
3196  for( size_t j=jj; j<jend; ++j ) {
3197  v_[i*nn_+j] *= (~rhs)(i,j);
3198  }
3199  }
3200  }
3201  }
3202 }
3203 //*************************************************************************************************
3204 
3205 
3206 //*************************************************************************************************
3217 template< typename Type // Data type of the matrix
3218  , bool AF // Alignment flag
3219  , bool PF // Padding flag
3220  , bool SO // Storage order
3221  , typename RT > // Result type
3222 template< typename MT > // Type of the right-hand side sparse matrix
3224 {
3225  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
3226  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
3227 
3228  const ResultType tmp( serial( *this ) );
3229 
3230  reset();
3231 
3232  for( size_t i=0UL; i<m_; ++i )
3233  for( auto element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
3234  v_[i*nn_+element->index()] = tmp(i,element->index()) * element->value();
3235 }
3236 //*************************************************************************************************
3237 
3238 
3239 //*************************************************************************************************
3250 template< typename Type // Data type of the matrix
3251  , bool AF // Alignment flag
3252  , bool PF // Padding flag
3253  , bool SO // Storage order
3254  , typename RT > // Result type
3255 template< typename MT > // Type of the right-hand side sparse matrix
3257 {
3259 
3260  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
3261  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
3262 
3263  const ResultType tmp( serial( *this ) );
3264 
3265  reset();
3266 
3267  for( size_t j=0UL; j<n_; ++j )
3268  for( auto element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
3269  v_[element->index()*nn_+j] = tmp(element->index(),j) * element->value();
3270 }
3271 //*************************************************************************************************
3272 
3273 
3274 
3275 
3276 
3277 
3278 
3279 
3280 //=================================================================================================
3281 //
3282 // CLASS TEMPLATE SPECIALIZATION FOR COLUMN-MAJOR MATRICES
3283 //
3284 //=================================================================================================
3285 
3286 //*************************************************************************************************
3294 template< typename Type // Data type of the matrix
3295  , bool AF // Alignment flag
3296  , bool PF // Padding flag
3297  , typename RT > // Result type
3298 class CustomMatrix<Type,AF,PF,true,RT>
3299  : public DenseMatrix< CustomMatrix<Type,AF,PF,true,RT>, true >
3300 {
3301  public:
3302  //**Type definitions****************************************************************************
3305 
3307  using ResultType = RT;
3308 
3311 
3314 
3315  using ElementType = Type;
3317  using ReturnType = const Type&;
3318  using CompositeType = const This&;
3319 
3320  using Reference = Type&;
3321  using ConstReference = const Type&;
3322  using Pointer = Type*;
3323  using ConstPointer = const Type*;
3324 
3327  //**********************************************************************************************
3328 
3329  //**Rebind struct definition********************************************************************
3332  template< typename NewType > // Data type of the other matrix
3333  struct Rebind {
3336  };
3337  //**********************************************************************************************
3338 
3339  //**Resize struct definition********************************************************************
3342  template< size_t NewM // Number of rows of the other matrix
3343  , size_t NewN > // Number of columns of the other matrix
3344  struct Resize {
3345  using RRT = Resize_t<RT,NewM,NewN>;
3346  using Other = CustomMatrix<Type,AF,PF,true,RRT>;
3347  };
3348  //**********************************************************************************************
3349 
3350  //**Compilation flags***************************************************************************
3352 
3356  static constexpr bool simdEnabled = IsVectorizable_v<Type>;
3357 
3359 
3362  static constexpr bool smpAssignable = !IsSMPAssignable_v<Type>;
3363  //**********************************************************************************************
3364 
3365  //**Constructors********************************************************************************
3368  explicit inline CustomMatrix();
3369  explicit inline CustomMatrix( Type* ptr, size_t m, size_t n );
3370  explicit inline CustomMatrix( Type* ptr, size_t m, size_t n, size_t mm );
3371 
3372  inline CustomMatrix( const CustomMatrix& m );
3373  inline CustomMatrix( CustomMatrix&& m ) noexcept;
3375  //**********************************************************************************************
3376 
3377  //**Destructor**********************************************************************************
3380  ~CustomMatrix() = default;
3382  //**********************************************************************************************
3383 
3384  //**Data access functions***********************************************************************
3387  inline Reference operator()( size_t i, size_t j ) noexcept;
3388  inline ConstReference operator()( size_t i, size_t j ) const noexcept;
3389  inline Reference at( size_t i, size_t j );
3390  inline ConstReference at( size_t i, size_t j ) const;
3391  inline Pointer data () noexcept;
3392  inline ConstPointer data () const noexcept;
3393  inline Pointer data ( size_t j ) noexcept;
3394  inline ConstPointer data ( size_t j ) const noexcept;
3395  inline Iterator begin ( size_t j ) noexcept;
3396  inline ConstIterator begin ( size_t j ) const noexcept;
3397  inline ConstIterator cbegin( size_t j ) const noexcept;
3398  inline Iterator end ( size_t j ) noexcept;
3399  inline ConstIterator end ( size_t j ) const noexcept;
3400  inline ConstIterator cend ( size_t j ) const noexcept;
3402  //**********************************************************************************************
3403 
3404  //**Assignment operators************************************************************************
3407  inline CustomMatrix& operator=( const Type& set );
3408  inline CustomMatrix& operator=( initializer_list< initializer_list<Type> > list );
3409 
3410  template< typename Other, size_t Rows, size_t Cols >
3411  inline CustomMatrix& operator=( const Other (&array)[Rows][Cols] );
3412 
3413  inline CustomMatrix& operator=( const CustomMatrix& rhs );
3414  inline CustomMatrix& operator=( CustomMatrix&& rhs ) noexcept;
3415 
3416  template< typename MT, bool SO > inline CustomMatrix& operator= ( const Matrix<MT,SO>& rhs );
3417  template< typename MT, bool SO > inline CustomMatrix& operator+=( const Matrix<MT,SO>& rhs );
3418  template< typename MT, bool SO > inline CustomMatrix& operator-=( const Matrix<MT,SO>& rhs );
3419  template< typename MT, bool SO > inline CustomMatrix& operator%=( const Matrix<MT,SO>& rhs );
3421  //**********************************************************************************************
3422 
3423  //**Utility functions***************************************************************************
3426  inline size_t rows() const noexcept;
3427  inline size_t columns() const noexcept;
3428  inline size_t spacing() const noexcept;
3429  inline size_t capacity() const noexcept;
3430  inline size_t capacity( size_t j ) const noexcept;
3431  inline size_t nonZeros() const;
3432  inline size_t nonZeros( size_t j ) const;
3433  inline void reset();
3434  inline void reset( size_t j );
3435  inline void clear();
3436  inline void swap( CustomMatrix& m ) noexcept;
3438  //**********************************************************************************************
3439 
3440  //**Numeric functions***************************************************************************
3443  inline CustomMatrix& transpose();
3444  inline CustomMatrix& ctranspose();
3445 
3446  template< typename Other > inline CustomMatrix& scale( const Other& scalar );
3448  //**********************************************************************************************
3449 
3450  //**Resource management functions***************************************************************
3453  inline void reset( Type* ptr, size_t m, size_t n );
3454  inline void reset( Type* ptr, size_t m, size_t n, size_t mm );
3456  //**********************************************************************************************
3457 
3458  private:
3459  //**********************************************************************************************
3461  template< typename MT >
3462  static constexpr bool VectorizedAssign_v =
3463  ( useOptimizedKernels &&
3464  simdEnabled && MT::simdEnabled &&
3465  IsSIMDCombinable_v< Type, ElementType_t<MT> > );
3466  //**********************************************************************************************
3467 
3468  //**********************************************************************************************
3470  template< typename MT >
3471  static constexpr bool VectorizedAddAssign_v =
3472  ( useOptimizedKernels &&
3473  simdEnabled && MT::simdEnabled &&
3474  IsSIMDCombinable_v< Type, ElementType_t<MT> > &&
3475  HasSIMDAdd_v< Type, ElementType_t<MT> > &&
3476  !IsDiagonal_v<MT> );
3477  //**********************************************************************************************
3478 
3479  //**********************************************************************************************
3481  template< typename MT >
3482  static constexpr bool VectorizedSubAssign_v =
3483  ( useOptimizedKernels &&
3484  simdEnabled && MT::simdEnabled &&
3485  IsSIMDCombinable_v< Type, ElementType_t<MT> > &&
3486  HasSIMDSub_v< Type, ElementType_t<MT> > &&
3487  !IsDiagonal_v<MT> );
3488  //**********************************************************************************************
3489 
3490  //**********************************************************************************************
3492  template< typename MT >
3493  static constexpr bool VectorizedSchurAssign_v =
3494  ( useOptimizedKernels &&
3495  simdEnabled && MT::simdEnabled &&
3496  IsSIMDCombinable_v< Type, ElementType_t<MT> > &&
3497  HasSIMDMult_v< Type, ElementType_t<MT> > );
3498  //**********************************************************************************************
3499 
3500  //**SIMD properties*****************************************************************************
3502  static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
3503  //**********************************************************************************************
3504 
3505  public:
3506  //**Expression template evaluation functions****************************************************
3509  template< typename Other > inline bool canAlias ( const Other* alias ) const noexcept;
3510  template< typename Other > inline bool isAliased( const Other* alias ) const noexcept;
3511 
3512  inline bool isAligned () const noexcept;
3513  inline bool canSMPAssign() const noexcept;
3514 
3515  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
3516  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
3517  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
3518 
3519  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
3520  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
3521  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
3522  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
3523 
3524  template< typename MT >
3525  inline auto assign( const DenseMatrix<MT,true>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT> >;
3526 
3527  template< typename MT >
3528  inline auto assign( const DenseMatrix<MT,true>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT> >;
3529 
3530  template< typename MT > inline void assign( const DenseMatrix<MT,false>& rhs );
3531  template< typename MT > inline void assign( const SparseMatrix<MT,true>& rhs );
3532  template< typename MT > inline void assign( const SparseMatrix<MT,false>& rhs );
3533 
3534  template< typename MT >
3535  inline auto addAssign( const DenseMatrix<MT,true>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT> >;
3536 
3537  template< typename MT >
3538  inline auto addAssign( const DenseMatrix<MT,true>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT> >;
3539 
3540  template< typename MT > inline void addAssign( const DenseMatrix<MT,false>& rhs );
3541  template< typename MT > inline void addAssign( const SparseMatrix<MT,true>& rhs );
3542  template< typename MT > inline void addAssign( const SparseMatrix<MT,false>& rhs );
3543 
3544  template< typename MT >
3545  inline auto subAssign ( const DenseMatrix<MT,true>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT> >;
3546 
3547  template< typename MT >
3548  inline auto subAssign ( const DenseMatrix<MT,true>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT> >;
3549 
3550  template< typename MT > inline void subAssign( const DenseMatrix<MT,false>& rhs );
3551  template< typename MT > inline void subAssign( const SparseMatrix<MT,true>& rhs );
3552  template< typename MT > inline void subAssign( const SparseMatrix<MT,false>& rhs );
3553 
3554  template< typename MT >
3555  inline auto schurAssign ( const DenseMatrix<MT,true>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT> >;
3556 
3557  template< typename MT >
3558  inline auto schurAssign ( const DenseMatrix<MT,true>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT> >;
3559 
3560  template< typename MT > inline void schurAssign( const DenseMatrix<MT,false>& rhs );
3561  template< typename MT > inline void schurAssign( const SparseMatrix<MT,true>& rhs );
3562  template< typename MT > inline void schurAssign( const SparseMatrix<MT,false>& rhs );
3564  //**********************************************************************************************
3565 
3566  private:
3567  //**Member variables****************************************************************************
3570  size_t m_;
3571  size_t mm_;
3572  size_t n_;
3573  Type* v_;
3574 
3577  //**********************************************************************************************
3578 
3579  //**Compile time checks*************************************************************************
3583  //**********************************************************************************************
3584 };
3586 //*************************************************************************************************
3587 
3588 
3589 
3590 
3591 //=================================================================================================
3592 //
3593 // CONSTRUCTORS
3594 //
3595 //=================================================================================================
3596 
3597 //*************************************************************************************************
3601 template< typename Type // Data type of the matrix
3602  , bool AF // Alignment flag
3603  , bool PF // Padding flag
3604  , typename RT > // Result type
3605 inline CustomMatrix<Type,AF,PF,true,RT>::CustomMatrix()
3606  : m_ ( 0UL ) // The current number of rows of the matrix
3607  , mm_( 0UL ) // The number of elements between two columns
3608  , n_ ( 0UL ) // The current number of columns of the matrix
3609  , v_ ( nullptr ) // The custom array of elements
3610 {}
3612 //*************************************************************************************************
3613 
3614 
3615 //*************************************************************************************************
3636 template< typename Type // Data type of the matrix
3637  , bool AF // Alignment flag
3638  , bool PF // Padding flag
3639  , typename RT > // Result type
3640 inline CustomMatrix<Type,AF,PF,true,RT>::CustomMatrix( Type* ptr, size_t m, size_t n )
3641  : m_ ( m ) // The current number of rows of the matrix
3642  , mm_( m ) // The number of elements between two columns
3643  , n_ ( n ) // The current number of columns of the matrix
3644  , v_ ( ptr ) // The custom array of elements
3645 {
3646  BLAZE_STATIC_ASSERT( PF == unpadded );
3647 
3648  if( ptr == nullptr ) {
3649  BLAZE_THROW_INVALID_ARGUMENT( "Invalid array of elements" );
3650  }
3651 
3652  if( AF && ( !checkAlignment( ptr ) || mm_ % SIMDSIZE != 0UL ) ) {
3653  BLAZE_THROW_INVALID_ARGUMENT( "Invalid alignment detected" );
3654  }
3655 }
3657 //*************************************************************************************************
3658 
3659 
3660 //*************************************************************************************************
3683 template< typename Type // Data type of the matrix
3684  , bool AF // Alignment flag
3685  , bool PF // Padding flag
3686  , typename RT > // Result type
3687 inline CustomMatrix<Type,AF,PF,true,RT>::CustomMatrix( Type* ptr, size_t m, size_t n, size_t mm )
3688  : m_ ( m ) // The current number of rows of the matrix
3689  , mm_( mm ) // The number of elements between two columns
3690  , n_ ( n ) // The current number of columns of the matrix
3691  , v_ ( ptr ) // The custom array of elements
3692 {
3693  using blaze::clear;
3694 
3695  using ClearFunctor = If_t< PF || !IsConst_v<Type>, Clear, Noop >;
3696 
3697  if( ptr == nullptr ) {
3698  BLAZE_THROW_INVALID_ARGUMENT( "Invalid array of elements" );
3699  }
3700 
3701  if( AF && ( !checkAlignment( ptr ) || mm_ % SIMDSIZE != 0UL ) ) {
3702  BLAZE_THROW_INVALID_ARGUMENT( "Invalid alignment detected" );
3703  }
3704 
3705  if( PF && IsVectorizable_v<Type> && ( mm_ < nextMultiple<size_t>( m_, SIMDSIZE ) ) ) {
3706  BLAZE_THROW_INVALID_ARGUMENT( "Insufficient capacity for padded matrix" );
3707  }
3708 
3709  if( PF && IsVectorizable_v<Type> ) {
3710  ClearFunctor clear;
3711  for( size_t j=0UL; j<n_; ++j )
3712  for( size_t i=m_; i<mm_; ++i ) {
3713  clear( v_[i+j*mm_] );
3714  }
3715  }
3716 }
3718 //*************************************************************************************************
3719 
3720 
3721 //*************************************************************************************************
3730 template< typename Type // Data type of the matrix
3731  , bool AF // Alignment flag
3732  , bool PF // Padding flag
3733  , typename RT > // Result type
3734 inline CustomMatrix<Type,AF,PF,true,RT>::CustomMatrix( const CustomMatrix& m )
3735  : m_ ( m.m_ ) // The current number of rows of the matrix
3736  , mm_( m.mm_ ) // The number of elements between two columns
3737  , n_ ( m.n_ ) // The current number of columns of the matrix
3738  , v_ ( m.v_ ) // The custom array of elements
3739 {}
3741 //*************************************************************************************************
3742 
3743 
3744 //*************************************************************************************************
3750 template< typename Type // Data type of the matrix
3751  , bool AF // Alignment flag
3752  , bool PF // Padding flag
3753  , typename RT > // Result type
3754 inline CustomMatrix<Type,AF,PF,true,RT>::CustomMatrix( CustomMatrix&& m ) noexcept
3755  : m_ ( m.m_ ) // The current number of rows of the matrix
3756  , mm_( m.mm_ ) // The number of elements between two columns
3757  , n_ ( m.n_ ) // The current number of columns of the matrix
3758  , v_ ( m.v_ ) // The custom array of elements
3759 {
3760  m.m_ = 0UL;
3761  m.mm_ = 0UL;
3762  m.n_ = 0UL;
3763  m.v_ = nullptr;
3764 
3765  BLAZE_INTERNAL_ASSERT( m.data() == nullptr, "Invalid data reference detected" );
3766 }
3768 //*************************************************************************************************
3769 
3770 
3771 
3772 
3773 //=================================================================================================
3774 //
3775 // DATA ACCESS FUNCTIONS
3776 //
3777 //=================================================================================================
3778 
3779 //*************************************************************************************************
3790 template< typename Type // Data type of the matrix
3791  , bool AF // Alignment flag
3792  , bool PF // Padding flag
3793  , typename RT > // Result type
3795  CustomMatrix<Type,AF,PF,true,RT>::operator()( size_t i, size_t j ) noexcept
3796 {
3797  BLAZE_USER_ASSERT( i<m_, "Invalid row access index" );
3798  BLAZE_USER_ASSERT( j<n_, "Invalid column access index" );
3799  return v_[i+j*mm_];
3800 }
3802 //*************************************************************************************************
3803 
3804 
3805 //*************************************************************************************************
3816 template< typename Type // Data type of the matrix
3817  , bool AF // Alignment flag
3818  , bool PF // Padding flag
3819  , typename RT > // Result type
3821  CustomMatrix<Type,AF,PF,true,RT>::operator()( size_t i, size_t j ) const noexcept
3822 {
3823  BLAZE_USER_ASSERT( i<m_, "Invalid row access index" );
3824  BLAZE_USER_ASSERT( j<n_, "Invalid column access index" );
3825  return v_[i+j*mm_];
3826 }
3828 //*************************************************************************************************
3829 
3830 
3831 //*************************************************************************************************
3843 template< typename Type // Data type of the matrix
3844  , bool AF // Alignment flag
3845  , bool PF // Padding flag
3846  , typename RT > // Result type
3848  CustomMatrix<Type,AF,PF,true,RT>::at( size_t i, size_t j )
3849 {
3850  if( i >= m_ ) {
3851  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
3852  }
3853  if( j >= n_ ) {
3854  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
3855  }
3856  return (*this)(i,j);
3857 }
3859 //*************************************************************************************************
3860 
3861 
3862 //*************************************************************************************************
3874 template< typename Type // Data type of the matrix
3875  , bool AF // Alignment flag
3876  , bool PF // Padding flag
3877  , typename RT > // Result type
3879  CustomMatrix<Type,AF,PF,true,RT>::at( size_t i, size_t j ) const
3880 {
3881  if( i >= m_ ) {
3882  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
3883  }
3884  if( j >= n_ ) {
3885  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
3886  }
3887  return (*this)(i,j);
3888 }
3890 //*************************************************************************************************
3891 
3892 
3893 //*************************************************************************************************
3905 template< typename Type // Data type of the matrix
3906  , bool AF // Alignment flag
3907  , bool PF // Padding flag
3908  , typename RT > // Result type
3911 {
3912  return v_;
3913 }
3915 //*************************************************************************************************
3916 
3917 
3918 //*************************************************************************************************
3930 template< typename Type // Data type of the matrix
3931  , bool AF // Alignment flag
3932  , bool PF // Padding flag
3933  , typename RT > // Result type
3936 {
3937  return v_;
3938 }
3940 //*************************************************************************************************
3941 
3942 
3943 //*************************************************************************************************
3952 template< typename Type // Data type of the matrix
3953  , bool AF // Alignment flag
3954  , bool PF // Padding flag
3955  , typename RT > // Result type
3957  CustomMatrix<Type,AF,PF,true,RT>::data( size_t j ) noexcept
3958 {
3959  BLAZE_USER_ASSERT( j < n_, "Invalid dense matrix column access index" );
3960  return v_+j*mm_;
3961 }
3963 //*************************************************************************************************
3964 
3965 
3966 //*************************************************************************************************
3975 template< typename Type // Data type of the matrix
3976  , bool AF // Alignment flag
3977  , bool PF // Padding flag
3978  , typename RT > // Result type
3980  CustomMatrix<Type,AF,PF,true,RT>::data( size_t j ) const noexcept
3981 {
3982  BLAZE_USER_ASSERT( j < n_, "Invalid dense matrix column access index" );
3983  return v_+j*mm_;
3984 }
3986 //*************************************************************************************************
3987 
3988 
3989 //*************************************************************************************************
3996 template< typename Type // Data type of the matrix
3997  , bool AF // Alignment flag
3998  , bool PF // Padding flag
3999  , typename RT > // Result type
4001  CustomMatrix<Type,AF,PF,true,RT>::begin( size_t j ) noexcept
4002 {
4003  BLAZE_USER_ASSERT( j < n_, "Invalid dense matrix column access index" );
4004  return Iterator( v_+j*mm_ );
4005 }
4007 //*************************************************************************************************
4008 
4009 
4010 //*************************************************************************************************
4017 template< typename Type // Data type of the matrix
4018  , bool AF // Alignment flag
4019  , bool PF // Padding flag
4020  , typename RT > // Result type
4022  CustomMatrix<Type,AF,PF,true,RT>::begin( size_t j ) const noexcept
4023 {
4024  BLAZE_USER_ASSERT( j < n_, "Invalid dense matrix column access index" );
4025  return ConstIterator( v_+j*mm_ );
4026 }
4028 //*************************************************************************************************
4029 
4030 
4031 //*************************************************************************************************
4038 template< typename Type // Data type of the matrix
4039  , bool AF // Alignment flag
4040  , bool PF // Padding flag
4041  , typename RT > // Result type
4043  CustomMatrix<Type,AF,PF,true,RT>::cbegin( size_t j ) const noexcept
4044 {
4045  BLAZE_USER_ASSERT( j < n_, "Invalid dense matrix column access index" );
4046  return ConstIterator( v_+j*mm_ );
4047 }
4049 //*************************************************************************************************
4050 
4051 
4052 //*************************************************************************************************
4059 template< typename Type // Data type of the matrix
4060  , bool AF // Alignment flag
4061  , bool PF // Padding flag
4062  , typename RT > // Result type
4064  CustomMatrix<Type,AF,PF,true,RT>::end( size_t j ) noexcept
4065 {
4066  BLAZE_USER_ASSERT( j < n_, "Invalid dense matrix column access index" );
4067  return Iterator( v_+j*mm_+m_ );
4068 }
4070 //*************************************************************************************************
4071 
4072 
4073 //*************************************************************************************************
4080 template< typename Type // Data type of the matrix
4081  , bool AF // Alignment flag
4082  , bool PF // Padding flag
4083  , typename RT > // Result type
4085  CustomMatrix<Type,AF,PF,true,RT>::end( size_t j ) const noexcept
4086 {
4087  BLAZE_USER_ASSERT( j < n_, "Invalid dense matrix column access index" );
4088  return ConstIterator( v_+j*mm_+m_ );
4089 }
4091 //*************************************************************************************************
4092 
4093 
4094 //*************************************************************************************************
4101 template< typename Type // Data type of the matrix
4102  , bool AF // Alignment flag
4103  , bool PF // Padding flag
4104  , typename RT > // Result type
4106  CustomMatrix<Type,AF,PF,true,RT>::cend( size_t j ) const noexcept
4107 {
4108  BLAZE_USER_ASSERT( j < n_, "Invalid dense matrix column access index" );
4109  return ConstIterator( v_+j*mm_+m_ );
4110 }
4112 //*************************************************************************************************
4113 
4114 
4115 
4116 
4117 //=================================================================================================
4118 //
4119 // ASSIGNMENT OPERATORS
4120 //
4121 //=================================================================================================
4122 
4123 //*************************************************************************************************
4130 template< typename Type // Data type of the matrix
4131  , bool AF // Alignment flag
4132  , bool PF // Padding flag
4133  , typename RT > // Result type
4134 inline CustomMatrix<Type,AF,PF,true,RT>&
4136 {
4137  for( size_t j=0UL; j<n_; ++j )
4138  for( size_t i=0UL; i<m_; ++i )
4139  v_[i+j*mm_] = rhs;
4140 
4141  return *this;
4142 }
4144 //*************************************************************************************************
4145 
4146 
4147 //*************************************************************************************************
4176 template< typename Type // Data type of the matrix
4177  , bool AF // Alignment flag
4178  , bool PF // Padding flag
4179  , typename RT > // Result type
4180 inline CustomMatrix<Type,AF,PF,true,RT>&
4181  CustomMatrix<Type,AF,PF,true,RT>::operator=( initializer_list< initializer_list<Type> > list )
4182 {
4183  using blaze::clear;
4184 
4185  if( list.size() != m_ || determineColumns( list ) > n_ ) {
4186  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to custom matrix" );
4187  }
4188 
4189  size_t i( 0UL );
4190 
4191  for( const auto& rowList : list ) {
4192  size_t j( 0UL );
4193  for( const auto& element : rowList ) {
4194  v_[i+j*mm_] = element;
4195  ++j;
4196  }
4197  for( ; j<n_; ++j ) {
4198  clear( v_[i+j*mm_] );
4199  }
4200  ++i;
4201  }
4202 
4203  return *this;
4204 }
4206 //*************************************************************************************************
4207 
4208 
4209 //*************************************************************************************************
4239 template< typename Type // Data type of the matrix
4240  , bool AF // Alignment flag
4241  , bool PF // Padding flag
4242  , typename RT > // Result type
4243 template< typename Other // Data type of the initialization array
4244  , size_t Rows // Number of rows of the initialization array
4245  , size_t Cols > // Number of columns of the initialization array
4246 inline CustomMatrix<Type,AF,PF,true,RT>&
4247  CustomMatrix<Type,AF,PF,true,RT>::operator=( const Other (&array)[Rows][Cols] )
4248 {
4249  if( m_ != Rows || n_ != Cols ) {
4250  BLAZE_THROW_INVALID_ARGUMENT( "Invalid array size" );
4251  }
4252 
4253  for( size_t j=0UL; j<Cols; ++j )
4254  for( size_t i=0UL; i<Rows; ++i )
4255  v_[i+j*mm_] = array[i][j];
4256 
4257  return *this;
4258 }
4260 //*************************************************************************************************
4261 
4262 
4263 //*************************************************************************************************
4274 template< typename Type // Data type of the matrix
4275  , bool AF // Alignment flag
4276  , bool PF // Padding flag
4277  , typename RT > // Result type
4278 inline CustomMatrix<Type,AF,PF,true,RT>&
4279  CustomMatrix<Type,AF,PF,true,RT>::operator=( const CustomMatrix& rhs )
4280 {
4281  if( rhs.rows() != m_ || rhs.columns() != n_ ) {
4282  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4283  }
4284 
4285  smpAssign( *this, ~rhs );
4286 
4287  return *this;
4288 }
4290 //*************************************************************************************************
4291 
4292 
4293 //*************************************************************************************************
4300 template< typename Type // Data type of the matrix
4301  , bool AF // Alignment flag
4302  , bool PF // Padding flag
4303  , typename RT > // Result type
4304 inline CustomMatrix<Type,AF,PF,true,RT>&
4305  CustomMatrix<Type,AF,PF,true,RT>::operator=( CustomMatrix&& rhs ) noexcept
4306 {
4307  m_ = rhs.m_;
4308  mm_ = rhs.mm_;
4309  n_ = rhs.n_;
4310  v_ = rhs.v_;
4311 
4312  rhs.m_ = 0UL;
4313  rhs.mm_ = 0UL;
4314  rhs.n_ = 0UL;
4315  rhs.v_ = nullptr;
4316 
4317  BLAZE_INTERNAL_ASSERT( rhs.data() == nullptr, "Invalid data reference detected" );
4318 
4319  return *this;
4320 }
4322 //*************************************************************************************************
4323 
4324 
4325 //*************************************************************************************************
4336 template< typename Type // Data type of the matrix
4337  , bool AF // Alignment flag
4338  , bool PF // Padding flag
4339  , typename RT > // Result type
4340 template< typename MT // Type of the right-hand side matrix
4341  , bool SO > // Storage order of the right-hand side matrix
4342 inline CustomMatrix<Type,AF,PF,true,RT>&
4343  CustomMatrix<Type,AF,PF,true,RT>::operator=( const Matrix<MT,SO>& rhs )
4344 {
4345  using TT = decltype( trans( *this ) );
4346  using CT = decltype( ctrans( *this ) );
4347  using IT = decltype( inv( *this ) );
4348 
4349  if( (~rhs).rows() != m_ || (~rhs).columns() != n_ ) {
4350  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4351  }
4352 
4353  if( IsSame_v<MT,TT> && (~rhs).isAliased( this ) ) {
4354  transpose();
4355  }
4356  else if( IsSame_v<MT,CT> && (~rhs).isAliased( this ) ) {
4357  ctranspose();
4358  }
4359  else if( !IsSame_v<MT,IT> && (~rhs).canAlias( this ) ) {
4360  const ResultType_t<MT> tmp( ~rhs );
4361  smpAssign( *this, tmp );
4362  }
4363  else {
4364  if( IsSparseMatrix_v<MT> )
4365  reset();
4366  smpAssign( *this, ~rhs );
4367  }
4368 
4369  return *this;
4370 }
4372 //*************************************************************************************************
4373 
4374 
4375 //*************************************************************************************************
4386 template< typename Type // Data type of the matrix
4387  , bool AF // Alignment flag
4388  , bool PF // Padding flag
4389  , typename RT > // Result type
4390 template< typename MT // Type of the right-hand side matrix
4391  , bool SO > // Storage order of the right-hand side matrix
4392 inline CustomMatrix<Type,AF,PF,true,RT>&
4393  CustomMatrix<Type,AF,PF,true,RT>::operator+=( const Matrix<MT,SO>& rhs )
4394 {
4395  if( (~rhs).rows() != m_ || (~rhs).columns() != n_ ) {
4396  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4397  }
4398 
4399  if( (~rhs).canAlias( this ) ) {
4400  const ResultType_t<MT> tmp( ~rhs );
4401  smpAddAssign( *this, tmp );
4402  }
4403  else {
4404  smpAddAssign( *this, ~rhs );
4405  }
4406 
4407  return *this;
4408 }
4410 //*************************************************************************************************
4411 
4412 
4413 //*************************************************************************************************
4424 template< typename Type // Data type of the matrix
4425  , bool AF // Alignment flag
4426  , bool PF // Padding flag
4427  , typename RT > // Result type
4428 template< typename MT // Type of the right-hand side matrix
4429  , bool SO > // Storage order of the right-hand side matrix
4430 inline CustomMatrix<Type,AF,PF,true,RT>&
4431  CustomMatrix<Type,AF,PF,true,RT>::operator-=( const Matrix<MT,SO>& rhs )
4432 {
4433  if( (~rhs).rows() != m_ || (~rhs).columns() != n_ ) {
4434  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4435  }
4436 
4437  if( (~rhs).canAlias( this ) ) {
4438  const ResultType_t<MT> tmp( ~rhs );
4439  smpSubAssign( *this, tmp );
4440  }
4441  else {
4442  smpSubAssign( *this, ~rhs );
4443  }
4444 
4445  return *this;
4446 }
4448 //*************************************************************************************************
4449 
4450 
4451 //*************************************************************************************************
4462 template< typename Type // Data type of the matrix
4463  , bool AF // Alignment flag
4464  , bool PF // Padding flag
4465  , typename RT > // Result type
4466 template< typename MT // Type of the right-hand side matrix
4467  , bool SO > // Storage order of the right-hand side matrix
4468 inline CustomMatrix<Type,AF,PF,true,RT>&
4469  CustomMatrix<Type,AF,PF,true,RT>::operator%=( const Matrix<MT,SO>& rhs )
4470 {
4471  if( (~rhs).rows() != m_ || (~rhs).columns() != n_ ) {
4472  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4473  }
4474 
4475  if( (~rhs).canAlias( this ) ) {
4476  const ResultType_t<MT> tmp( ~rhs );
4477  smpSchurAssign( *this, tmp );
4478  }
4479  else {
4480  smpSchurAssign( *this, ~rhs );
4481  }
4482 
4483  return *this;
4484 }
4486 //*************************************************************************************************
4487 
4488 
4489 
4490 
4491 //=================================================================================================
4492 //
4493 // UTILITY FUNCTIONS
4494 //
4495 //=================================================================================================
4496 
4497 //*************************************************************************************************
4503 template< typename Type // Data type of the matrix
4504  , bool AF // Alignment flag
4505  , bool PF // Padding flag
4506  , typename RT > // Result type
4507 inline size_t CustomMatrix<Type,AF,PF,true,RT>::rows() const noexcept
4508 {
4509  return m_;
4510 }
4512 //*************************************************************************************************
4513 
4514 
4515 //*************************************************************************************************
4521 template< typename Type // Data type of the matrix
4522  , bool AF // Alignment flag
4523  , bool PF // Padding flag
4524  , typename RT > // Result type
4525 inline size_t CustomMatrix<Type,AF,PF,true,RT>::columns() const noexcept
4526 {
4527  return n_;
4528 }
4530 //*************************************************************************************************
4531 
4532 
4533 //*************************************************************************************************
4542 template< typename Type // Data type of the matrix
4543  , bool AF // Alignment flag
4544  , bool PF // Padding flag
4545  , typename RT > // Result type
4546 inline size_t CustomMatrix<Type,AF,PF,true,RT>::spacing() const noexcept
4547 {
4548  return mm_;
4549 }
4551 //*************************************************************************************************
4552 
4553 
4554 //*************************************************************************************************
4560 template< typename Type // Data type of the matrix
4561  , bool AF // Alignment flag
4562  , bool PF // Padding flag
4563  , typename RT > // Result type
4564 inline size_t CustomMatrix<Type,AF,PF,true,RT>::capacity() const noexcept
4565 {
4566  return mm_ * n_;
4567 }
4569 //*************************************************************************************************
4570 
4571 
4572 //*************************************************************************************************
4579 template< typename Type // Data type of the matrix
4580  , bool AF // Alignment flag
4581  , bool PF // Padding flag
4582  , typename RT > // Result type
4583 inline size_t CustomMatrix<Type,AF,PF,true,RT>::capacity( size_t j ) const noexcept
4584 {
4585  MAYBE_UNUSED( j );
4586  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
4587  return mm_;
4588 }
4590 //*************************************************************************************************
4591 
4592 
4593 //*************************************************************************************************
4599 template< typename Type // Data type of the matrix
4600  , bool AF // Alignment flag
4601  , bool PF // Padding flag
4602  , typename RT > // Result type
4603 inline size_t CustomMatrix<Type,AF,PF,true,RT>::nonZeros() const
4604 {
4605  size_t nonzeros( 0UL );
4606 
4607  for( size_t j=0UL; j<n_; ++j )
4608  for( size_t i=0UL; i<m_; ++i )
4609  if( !isDefault( v_[i+j*mm_] ) )
4610  ++nonzeros;
4611 
4612  return nonzeros;
4613 }
4615 //*************************************************************************************************
4616 
4617 
4618 //*************************************************************************************************
4625 template< typename Type // Data type of the matrix
4626  , bool AF // Alignment flag
4627  , bool PF // Padding flag
4628  , typename RT > // Result type
4629 inline size_t CustomMatrix<Type,AF,PF,true,RT>::nonZeros( size_t j ) const
4630 {
4631  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
4632 
4633  const size_t iend( j*mm_ + m_ );
4634  size_t nonzeros( 0UL );
4635 
4636  for( size_t i=j*mm_; i<iend; ++i )
4637  if( !isDefault( v_[i] ) )
4638  ++nonzeros;
4639 
4640  return nonzeros;
4641 }
4643 //*************************************************************************************************
4644 
4645 
4646 //*************************************************************************************************
4652 template< typename Type // Data type of the matrix
4653  , bool AF // Alignment flag
4654  , bool PF // Padding flag
4655  , typename RT > // Result type
4657 {
4658  using blaze::clear;
4659 
4660  for( size_t j=0UL; j<n_; ++j )
4661  for( size_t i=0UL; i<m_; ++i )
4662  clear( v_[i+j*mm_] );
4663 }
4665 //*************************************************************************************************
4666 
4667 
4668 //*************************************************************************************************
4678 template< typename Type // Data type of the matrix
4679  , bool AF // Alignment flag
4680  , bool PF // Padding flag
4681  , typename RT > // Result type
4682 inline void CustomMatrix<Type,AF,PF,true,RT>::reset( size_t j )
4683 {
4684  using blaze::clear;
4685 
4686  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
4687  for( size_t i=0UL; i<m_; ++i )
4688  clear( v_[i+j*mm_] );
4689 }
4691 //*************************************************************************************************
4692 
4693 
4694 //*************************************************************************************************
4702 template< typename Type // Data type of the matrix
4703  , bool AF // Alignment flag
4704  , bool PF // Padding flag
4705  , typename RT > // Result type
4707 {
4708  m_ = 0UL;
4709  mm_ = 0UL;
4710  n_ = 0UL;
4711  v_ = nullptr;
4712 }
4714 //*************************************************************************************************
4715 
4716 
4717 //*************************************************************************************************
4724 template< typename Type // Data type of the matrix
4725  , bool AF // Alignment flag
4726  , bool PF // Padding flag
4727  , typename RT > // Result type
4728 inline void CustomMatrix<Type,AF,PF,true,RT>::swap( CustomMatrix& m ) noexcept
4729 {
4730  using std::swap;
4731 
4732  swap( m_ , m.m_ );
4733  swap( mm_, m.mm_ );
4734  swap( n_ , m.n_ );
4735  swap( v_ , m.v_ );
4736 }
4738 //*************************************************************************************************
4739 
4740 
4741 
4742 
4743 //=================================================================================================
4744 //
4745 // NUMERIC FUNCTIONS
4746 //
4747 //=================================================================================================
4748 
4749 //*************************************************************************************************
4758 template< typename Type // Data type of the matrix
4759  , bool AF // Alignment flag
4760  , bool PF // Padding flag
4761  , typename RT > // Result type
4762 inline CustomMatrix<Type,AF,PF,true,RT>& CustomMatrix<Type,AF,PF,true,RT>::transpose()
4763 {
4764  using std::swap;
4765 
4766  if( m_ != n_ ) {
4767  BLAZE_THROW_LOGIC_ERROR( "Impossible transpose operation" );
4768  }
4769 
4770  for( size_t j=1UL; j<n_; ++j )
4771  for( size_t i=0UL; i<j; ++i )
4772  swap( v_[i+j*mm_], v_[j+i*mm_] );
4773 
4774  return *this;
4775 }
4777 //*************************************************************************************************
4778 
4779 
4780 //*************************************************************************************************
4789 template< typename Type // Data type of the matrix
4790  , bool AF // Alignment flag
4791  , bool PF // Padding flag
4792  , typename RT > // Result type
4793 inline CustomMatrix<Type,AF,PF,true,RT>& CustomMatrix<Type,AF,PF,true,RT>::ctranspose()
4794 {
4795  if( m_ != n_ ) {
4796  BLAZE_THROW_LOGIC_ERROR( "Impossible transpose operation" );
4797  }
4798 
4799  for( size_t j=0UL; j<n_; ++j ) {
4800  for( size_t i=0UL; i<j; ++i ) {
4801  cswap( v_[i+j*mm_], v_[j+i*mm_] );
4802  }
4803  conjugate( v_[j+j*mm_] );
4804  }
4805 
4806  return *this;
4807 }
4809 //*************************************************************************************************
4810 
4811 
4812 //*************************************************************************************************
4834 template< typename Type // Data type of the matrix
4835  , bool AF // Alignment flag
4836  , bool PF // Padding flag
4837  , typename RT > // Result type
4838 template< typename Other > // Data type of the scalar value
4839 inline CustomMatrix<Type,AF,PF,true,RT>& CustomMatrix<Type,AF,PF,true,RT>::scale( const Other& scalar )
4840 {
4841  for( size_t j=0UL; j<n_; ++j )
4842  for( size_t i=0UL; i<m_; ++i )
4843  v_[i+j*mm_] *= scalar;
4844 
4845  return *this;
4846 }
4848 //*************************************************************************************************
4849 
4850 
4851 
4852 
4853 //=================================================================================================
4854 //
4855 // RESOURCE MANAGEMENT FUNCTIONS
4856 //
4857 //=================================================================================================
4858 
4859 //*************************************************************************************************
4883 template< typename Type // Data type of the matrix
4884  , bool AF // Alignment flag
4885  , bool PF // Padding flag
4886  , typename RT > // Result type
4887 inline void CustomMatrix<Type,AF,PF,true,RT>::reset( Type* ptr, size_t m, size_t n )
4888 {
4889  BLAZE_STATIC_ASSERT( PF == unpadded );
4890 
4891  CustomMatrix tmp( ptr, m, n );
4892  swap( tmp );
4893 }
4895 //*************************************************************************************************
4896 
4897 
4898 //*************************************************************************************************
4922 template< typename Type // Data type of the matrix
4923  , bool AF // Alignment flag
4924  , bool PF // Padding flag
4925  , typename RT > // Result type
4926 inline void CustomMatrix<Type,AF,PF,true,RT>::reset( Type* ptr, size_t m, size_t n, size_t mm )
4927 {
4928  CustomMatrix tmp( ptr, m, n, mm );
4929  swap( tmp );
4930 }
4932 //*************************************************************************************************
4933 
4934 
4935 
4936 
4937 //=================================================================================================
4938 //
4939 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
4940 //
4941 //=================================================================================================
4942 
4943 //*************************************************************************************************
4954 template< typename Type // Data type of the matrix
4955  , bool AF // Alignment flag
4956  , bool PF // Padding flag
4957  , typename RT > // Result type
4958 template< typename Other > // Data type of the foreign expression
4959 inline bool CustomMatrix<Type,AF,PF,true,RT>::canAlias( const Other* alias ) const noexcept
4960 {
4961  return static_cast<const void*>( this ) == static_cast<const void*>( alias );
4962 }
4964 //*************************************************************************************************
4965 
4966 
4967 //*************************************************************************************************
4978 template< typename Type // Data type of the matrix
4979  , bool AF // Alignment flag
4980  , bool PF // Padding flag
4981  , typename RT > // Result type
4982 template< typename Other > // Data type of the foreign expression
4983 inline bool CustomMatrix<Type,AF,PF,true,RT>::isAliased( const Other* alias ) const noexcept
4984 {
4985  return static_cast<const void*>( this ) == static_cast<const void*>( alias );
4986 }
4988 //*************************************************************************************************
4989 
4990 
4991 //*************************************************************************************************
5001 template< typename Type // Data type of the matrix
5002  , bool AF // Alignment flag
5003  , bool PF // Padding flag
5004  , typename RT > // Result type
5005 inline bool CustomMatrix<Type,AF,PF,true,RT>::isAligned() const noexcept
5006 {
5007  return ( AF || ( checkAlignment( v_ ) && rows() % SIMDSIZE == 0UL ) );
5008 }
5010 //*************************************************************************************************
5011 
5012 
5013 //*************************************************************************************************
5024 template< typename Type // Data type of the matrix
5025  , bool AF // Alignment flag
5026  , bool PF // Padding flag
5027  , typename RT > // Result type
5028 inline bool CustomMatrix<Type,AF,PF,true,RT>::canSMPAssign() const noexcept
5029 {
5030  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
5031 }
5033 //*************************************************************************************************
5034 
5035 
5036 //*************************************************************************************************
5051 template< typename Type // Data type of the matrix
5052  , bool AF // Alignment flag
5053  , bool PF // Padding flag
5054  , typename RT > // Result type
5056  CustomMatrix<Type,AF,PF,true,RT>::load( size_t i, size_t j ) const noexcept
5057 {
5058  if( AF && PF )
5059  return loada( i, j );
5060  else
5061  return loadu( i, j );
5062 }
5064 //*************************************************************************************************
5065 
5066 
5067 //*************************************************************************************************
5082 template< typename Type // Data type of the matrix
5083  , bool AF // Alignment flag
5084  , bool PF // Padding flag
5085  , typename RT > // Result type
5087  CustomMatrix<Type,AF,PF,true,RT>::loada( size_t i, size_t j ) const noexcept
5088 {
5089  using blaze::loada;
5090 
5092 
5093  BLAZE_INTERNAL_ASSERT( i < m_, "Invalid row access index" );
5094  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= ( PF ? mm_ : m_ ), "Invalid row access index" );
5095  BLAZE_INTERNAL_ASSERT( !PF || i % SIMDSIZE == 0UL, "Invalid row access index" );
5096  BLAZE_INTERNAL_ASSERT( j < n_, "Invalid column access index" );
5097  BLAZE_INTERNAL_ASSERT( checkAlignment( v_+i+j*mm_ ), "Invalid alignment detected" );
5098 
5099  return loada( v_+i+j*mm_ );
5100 }
5102 //*************************************************************************************************
5103 
5104 
5105 //*************************************************************************************************
5120 template< typename Type // Data type of the matrix
5121  , bool AF // Alignment flag
5122  , bool PF // Padding flag
5123  , typename RT > // Result type
5125  CustomMatrix<Type,AF,PF,true,RT>::loadu( size_t i, size_t j ) const noexcept
5126 {
5127  using blaze::loadu;
5128 
5130 
5131  BLAZE_INTERNAL_ASSERT( i < m_, "Invalid row access index" );
5132  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= ( PF ? mm_ : m_ ), "Invalid row access index" );
5133  BLAZE_INTERNAL_ASSERT( j < n_, "Invalid column access index" );
5134 
5135  return loadu( v_+i+j*mm_ );
5136 }
5138 //*************************************************************************************************
5139 
5140 
5141 //*************************************************************************************************
5157 template< typename Type // Data type of the matrix
5158  , bool AF // Alignment flag
5159  , bool PF // Padding flag
5160  , typename RT > // Result type
5162  CustomMatrix<Type,AF,PF,true,RT>::store( size_t i, size_t j, const SIMDType& value ) noexcept
5163 {
5164  if( AF && PF )
5165  storea( i, j, value );
5166  else
5167  storeu( i, j, value );
5168 }
5170 //*************************************************************************************************
5171 
5172 
5173 //*************************************************************************************************
5189 template< typename Type // Data type of the matrix
5190  , bool AF // Alignment flag
5191  , bool PF // Padding flag
5192  , typename RT > // Result type
5194  CustomMatrix<Type,AF,PF,true,RT>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
5195 {
5196  using blaze::storea;
5197 
5199 
5200  BLAZE_INTERNAL_ASSERT( i < m_, "Invalid row access index" );
5201  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= ( PF ? mm_ : m_ ), "Invalid row access index" );
5202  BLAZE_INTERNAL_ASSERT( !PF || i % SIMDSIZE == 0UL, "Invalid row access index" );
5203  BLAZE_INTERNAL_ASSERT( j < n_, "Invalid column access index" );
5204  BLAZE_INTERNAL_ASSERT( checkAlignment( v_+i+j*mm_ ), "Invalid alignment detected" );
5205 
5206  storea( v_+i+j*mm_, value );
5207 }
5209 //*************************************************************************************************
5210 
5211 
5212 //*************************************************************************************************
5228 template< typename Type // Data type of the matrix
5229  , bool AF // Alignment flag
5230  , bool PF // Padding flag
5231  , typename RT > // Result type
5233  CustomMatrix<Type,AF,PF,true,RT>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
5234 {
5235  using blaze::storeu;
5236 
5238 
5239  BLAZE_INTERNAL_ASSERT( i < m_, "Invalid row access index" );
5240  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= ( PF ? mm_ : m_ ), "Invalid row access index" );
5241  BLAZE_INTERNAL_ASSERT( j < n_, "Invalid column access index" );
5242 
5243  storeu( v_+i+j*mm_, value );
5244 }
5246 //*************************************************************************************************
5247 
5248 
5249 //*************************************************************************************************
5266 template< typename Type // Data type of the matrix
5267  , bool AF // Alignment flag
5268  , bool PF // Padding flag
5269  , typename RT > // Result type
5271  CustomMatrix<Type,AF,PF,true,RT>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
5272 {
5273  using blaze::stream;
5274 
5276 
5277  BLAZE_INTERNAL_ASSERT( i < m_, "Invalid row access index" );
5278  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= ( PF ? mm_ : m_ ), "Invalid row access index" );
5279  BLAZE_INTERNAL_ASSERT( !PF || i % SIMDSIZE == 0UL, "Invalid row access index" );
5280  BLAZE_INTERNAL_ASSERT( j < n_, "Invalid column access index" );
5281  BLAZE_INTERNAL_ASSERT( checkAlignment( v_+i+j*mm_ ), "Invalid alignment detected" );
5282 
5283  stream( v_+i+j*mm_, value );
5284 }
5286 //*************************************************************************************************
5287 
5288 
5289 //*************************************************************************************************
5301 template< typename Type // Data type of the matrix
5302  , bool AF // Alignment flag
5303  , bool PF // Padding flag
5304  , typename RT > // Result type
5305 template< typename MT > // Type of the right-hand side dense matrix
5306 inline auto CustomMatrix<Type,AF,PF,true,RT>::assign( const DenseMatrix<MT,true>& rhs )
5307  -> DisableIf_t< VectorizedAssign_v<MT> >
5308 {
5309  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5310  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5311 
5312  const size_t ipos( m_ & size_t(-2) );
5313  BLAZE_INTERNAL_ASSERT( ( m_ - ( m_ % 2UL ) ) == ipos, "Invalid end calculation" );
5314 
5315  for( size_t j=0UL; j<n_; ++j ) {
5316  for( size_t i=0UL; i<ipos; i+=2UL ) {
5317  v_[i +j*mm_] = (~rhs)(i ,j);
5318  v_[i+1UL+j*mm_] = (~rhs)(i+1UL,j);
5319  }
5320  if( ipos < m_ ) {
5321  v_[ipos+j*mm_] = (~rhs)(ipos,j);
5322  }
5323  }
5324 }
5326 //*************************************************************************************************
5327 
5328 
5329 //*************************************************************************************************
5341 template< typename Type // Data type of the matrix
5342  , bool AF // Alignment flag
5343  , bool PF // Padding flag
5344  , typename RT > // Result type
5345 template< typename MT > // Type of the right-hand side dense matrix
5346 inline auto CustomMatrix<Type,AF,PF,true,RT>::assign( const DenseMatrix<MT,true>& rhs )
5347  -> EnableIf_t< VectorizedAssign_v<MT> >
5348 {
5350 
5351  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5352  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5353 
5354  constexpr bool remainder( !PF || !IsPadded_v<MT> );
5355 
5356  const size_t ipos( ( remainder )?( m_ & size_t(-SIMDSIZE) ):( m_ ) );
5357  BLAZE_INTERNAL_ASSERT( !remainder || ( m_ - ( m_ % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
5358 
5359  if( AF && PF && useStreaming &&
5360  ( m_*n_ > ( cacheSize / ( sizeof(Type) * 3UL ) ) ) && !(~rhs).isAliased( this ) )
5361  {
5362  for( size_t j=0UL; j<n_; ++j )
5363  {
5364  size_t i( 0UL );
5365  Iterator left( begin(j) );
5366  ConstIterator_t<MT> right( (~rhs).begin(j) );
5367 
5368  for( ; i<ipos; i+=SIMDSIZE ) {
5369  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5370  }
5371  for( ; remainder && i<m_; ++i ) {
5372  *left = *right; ++left; ++right;
5373  }
5374  }
5375  }
5376  else
5377  {
5378  for( size_t j=0UL; j<n_; ++j )
5379  {
5380  size_t i( 0UL );
5381  Iterator left( begin(j) );
5382  ConstIterator_t<MT> right( (~rhs).begin(j) );
5383 
5384  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
5385  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5386  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5387  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5388  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5389  }
5390  for( ; i<ipos; i+=SIMDSIZE ) {
5391  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5392  }
5393  for( ; remainder && i<m_; ++i ) {
5394  *left = *right; ++left; ++right;
5395  }
5396  }
5397  }
5398 }
5400 //*************************************************************************************************
5401 
5402 
5403 //*************************************************************************************************
5415 template< typename Type // Data type of the matrix
5416  , bool AF // Alignment flag
5417  , bool PF // Padding flag
5418  , typename RT > // Result type
5419 template< typename MT > // Type of the right-hand side dense matrix
5420 inline void CustomMatrix<Type,AF,PF,true,RT>::assign( const DenseMatrix<MT,false>& rhs )
5421 {
5423 
5424  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5425  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5426 
5427  constexpr size_t block( BLOCK_SIZE );
5428 
5429  for( size_t jj=0UL; jj<n_; jj+=block ) {
5430  const size_t jend( min( n_, jj+block ) );
5431  for( size_t ii=0UL; ii<m_; ii+=block ) {
5432  const size_t iend( min( m_, ii+block ) );
5433  for( size_t j=jj; j<jend; ++j ) {
5434  for( size_t i=ii; i<iend; ++i ) {
5435  v_[i+j*mm_] = (~rhs)(i,j);
5436  }
5437  }
5438  }
5439  }
5440 }
5442 //*************************************************************************************************
5443 
5444 
5445 //*************************************************************************************************
5457 template< typename Type // Data type of the matrix
5458  , bool AF // Alignment flag
5459  , bool PF // Padding flag
5460  , typename RT > // Result type
5461 template< typename MT > // Type of the right-hand side sparse matrix
5462 inline void CustomMatrix<Type,AF,PF,true,RT>::assign( const SparseMatrix<MT,true>& rhs )
5463 {
5464  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5465  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5466 
5467  for( size_t j=0UL; j<(~rhs).columns(); ++j )
5468  for( auto element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
5469  v_[element->index()+j*mm_] = element->value();
5470 }
5472 //*************************************************************************************************
5473 
5474 
5475 //*************************************************************************************************
5487 template< typename Type // Data type of the matrix
5488  , bool AF // Alignment flag
5489  , bool PF // Padding flag
5490  , typename RT > // Result type
5491 template< typename MT > // Type of the right-hand side sparse matrix
5492 inline void CustomMatrix<Type,AF,PF,true,RT>::assign( const SparseMatrix<MT,false>& rhs )
5493 {
5495 
5496  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5497  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5498 
5499  for( size_t i=0UL; i<(~rhs).rows(); ++i )
5500  for( auto element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
5501  v_[i+element->index()*mm_] = element->value();
5502 }
5504 //*************************************************************************************************
5505 
5506 
5507 //*************************************************************************************************
5519 template< typename Type // Data type of the matrix
5520  , bool AF // Alignment flag
5521  , bool PF // Padding flag
5522  , typename RT > // Result type
5523 template< typename MT > // Type of the right-hand side dense matrix
5524 inline auto CustomMatrix<Type,AF,PF,true,RT>::addAssign( const DenseMatrix<MT,true>& rhs )
5525  -> DisableIf_t< VectorizedAddAssign_v<MT> >
5526 {
5527  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5528  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5529 
5530  for( size_t j=0UL; j<n_; ++j )
5531  {
5532  if( IsDiagonal_v<MT> )
5533  {
5534  v_[j+j*mm_] += (~rhs)(j,j);
5535  }
5536  else
5537  {
5538  const size_t ibegin( ( IsLower_v<MT> )
5539  ?( IsStrictlyLower_v<MT> ? j+1UL : j )
5540  :( 0UL ) );
5541  const size_t iend ( ( IsUpper_v<MT> )
5542  ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
5543  :( m_ ) );
5544  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
5545 
5546  size_t i( ibegin );
5547 
5548  for( ; (i+2UL) <= iend; i+=2UL ) {
5549  v_[i +j*mm_] += (~rhs)(i ,j);
5550  v_[i+1UL+j*mm_] += (~rhs)(i+1UL,j);
5551  }
5552  if( i < iend ) {
5553  v_[i+j*mm_] += (~rhs)(i,j);
5554  }
5555  }
5556  }
5557 }
5559 //*************************************************************************************************
5560 
5561 
5562 //*************************************************************************************************
5574 template< typename Type // Data type of the matrix
5575  , bool AF // Alignment flag
5576  , bool PF // Padding flag
5577  , typename RT > // Result type
5578 template< typename MT > // Type of the right-hand side dense matrix
5579 inline auto CustomMatrix<Type,AF,PF,true,RT>::addAssign( const DenseMatrix<MT,true>& rhs )
5580  -> EnableIf_t< VectorizedAddAssign_v<MT> >
5581 {
5584 
5585  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5586  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5587 
5588  constexpr bool remainder( !PF || !IsPadded_v<MT> );
5589 
5590  for( size_t j=0UL; j<n_; ++j )
5591  {
5592  const size_t ibegin( ( IsLower_v<MT> )
5593  ?( ( IsStrictlyLower_v<MT> ? j+1UL : j ) & size_t(-SIMDSIZE) )
5594  :( 0UL ) );
5595  const size_t iend ( ( IsUpper_v<MT> )
5596  ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
5597  :( m_ ) );
5598  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
5599 
5600  const size_t ipos( ( remainder )?( iend & size_t(-SIMDSIZE) ):( iend ) );
5601  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
5602 
5603  size_t i( ibegin );
5604  Iterator left( begin(j) + ibegin );
5605  ConstIterator_t<MT> right( (~rhs).begin(j) + ibegin );
5606 
5607  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
5608  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5609  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5610  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5611  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5612  }
5613  for( ; i<ipos; i+=SIMDSIZE ) {
5614  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5615  }
5616  for( ; remainder && i<iend; ++i ) {
5617  *left += *right; ++left; ++right;
5618  }
5619  }
5620 }
5622 //*************************************************************************************************
5623 
5624 
5625 //*************************************************************************************************
5637 template< typename Type // Data type of the matrix
5638  , bool AF // Alignment flag
5639  , bool PF // Padding flag
5640  , typename RT > // Result type
5641 template< typename MT > // Type of the right-hand side dense matrix
5642 inline void CustomMatrix<Type,AF,PF,true,RT>::addAssign( const DenseMatrix<MT,false>& rhs )
5643 {
5645 
5646  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5647  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5648 
5649  constexpr size_t block( BLOCK_SIZE );
5650 
5651  for( size_t jj=0UL; jj<n_; jj+=block ) {
5652  const size_t jend( min( n_, jj+block ) );
5653  for( size_t ii=0UL; ii<m_; ii+=block )
5654  {
5655  if( IsLower_v<MT> && ii < jj ) continue;
5656  if( IsUpper_v<MT> && ii > jj ) break;
5657 
5658  for( size_t j=jj; j<jend; ++j )
5659  {
5660  const size_t ibegin( ( IsLower_v<MT> )
5661  ?( max( ( IsStrictlyLower_v<MT> ? j+1UL : j ), ii ) )
5662  :( ii ) );
5663  const size_t iend ( ( IsUpper_v<MT> )
5664  ?( min( ( IsStrictlyUpper_v<MT> ? j : j+1UL ), m_, ii+block ) )
5665  :( min( m_, ii+block ) ) );
5666  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
5667 
5668  for( size_t i=ibegin; i<iend; ++i ) {
5669  v_[i+j*mm_] += (~rhs)(i,j);
5670  }
5671  }
5672  }
5673  }
5674 }
5676 //*************************************************************************************************
5677 
5678 
5679 //*************************************************************************************************
5691 template< typename Type // Data type of the matrix
5692  , bool AF // Alignment flag
5693  , bool PF // Padding flag
5694  , typename RT > // Result type
5695 template< typename MT > // Type of the right-hand side sparse matrix
5696 inline void CustomMatrix<Type,AF,PF,true,RT>::addAssign( const SparseMatrix<MT,true>& rhs )
5697 {
5698  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5699  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5700 
5701  for( size_t j=0UL; j<(~rhs).columns(); ++j )
5702  for( auto element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
5703  v_[element->index()+j*mm_] += element->value();
5704 }
5706 //*************************************************************************************************
5707 
5708 
5709 //*************************************************************************************************
5721 template< typename Type // Data type of the matrix
5722  , bool AF // Alignment flag
5723  , bool PF // Padding flag
5724  , typename RT > // Result type
5725 template< typename MT > // Type of the right-hand side sparse matrix
5726 inline void CustomMatrix<Type,AF,PF,true,RT>::addAssign( const SparseMatrix<MT,false>& rhs )
5727 {
5729 
5730  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5731  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5732 
5733  for( size_t i=0UL; i<(~rhs).rows(); ++i )
5734  for( auto element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
5735  v_[i+element->index()*mm_] += element->value();
5736 }
5738 //*************************************************************************************************
5739 
5740 
5741 //*************************************************************************************************
5753 template< typename Type // Data type of the matrix
5754  , bool AF // Alignment flag
5755  , bool PF // Padding flag
5756  , typename RT > // Result type
5757 template< typename MT > // Type of the right-hand side dense matrix
5758 inline auto CustomMatrix<Type,AF,PF,true,RT>::subAssign( const DenseMatrix<MT,true>& rhs )
5759  -> DisableIf_t< VectorizedSubAssign_v<MT> >
5760 {
5761  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5762  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5763 
5764  for( size_t j=0UL; j<n_; ++j )
5765  {
5766  if( IsDiagonal_v<MT> )
5767  {
5768  v_[j+j*mm_] -= (~rhs)(j,j);
5769  }
5770  else
5771  {
5772  const size_t ibegin( ( IsLower_v<MT> )
5773  ?( IsStrictlyLower_v<MT> ? j+1UL : j )
5774  :( 0UL ) );
5775  const size_t iend ( ( IsUpper_v<MT> )
5776  ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
5777  :( m_ ) );
5778  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
5779 
5780  size_t i( ibegin );
5781 
5782  for( ; (i+2UL) <= iend; i+=2UL ) {
5783  v_[i +j*mm_] -= (~rhs)(i ,j);
5784  v_[i+1+j*mm_] -= (~rhs)(i+1,j);
5785  }
5786  if( i < iend ) {
5787  v_[i+j*mm_] -= (~rhs)(i,j);
5788  }
5789  }
5790  }
5791 }
5793 //*************************************************************************************************
5794 
5795 
5796 //*************************************************************************************************
5809 template< typename Type // Data type of the matrix
5810  , bool AF // Alignment flag
5811  , bool PF // Padding flag
5812  , typename RT > // Result type
5813 template< typename MT > // Type of the right-hand side dense matrix
5814 inline auto CustomMatrix<Type,AF,PF,true,RT>::subAssign( const DenseMatrix<MT,true>& rhs )
5815  -> EnableIf_t< VectorizedSubAssign_v<MT> >
5816 {
5819 
5820  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5821  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5822 
5823  constexpr bool remainder( !PF || !IsPadded_v<MT> );
5824 
5825  for( size_t j=0UL; j<n_; ++j )
5826  {
5827  const size_t ibegin( ( IsLower_v<MT> )
5828  ?( ( IsStrictlyLower_v<MT> ? j+1UL : j ) & size_t(-SIMDSIZE) )
5829  :( 0UL ) );
5830  const size_t iend ( ( IsUpper_v<MT> )
5831  ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
5832  :( m_ ) );
5833  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
5834 
5835  const size_t ipos( ( remainder )?( iend & size_t(-SIMDSIZE) ):( iend ) );
5836  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
5837 
5838  size_t i( ibegin );
5839  Iterator left( begin(j) + ibegin );
5840  ConstIterator_t<MT> right( (~rhs).begin(j) + ibegin );
5841 
5842  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
5843  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5844  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5845  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5846  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5847  }
5848  for( ; i<ipos; i+=SIMDSIZE ) {
5849  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5850  }
5851  for( ; remainder && i<iend; ++i ) {
5852  *left -= *right; ++left; ++right;
5853  }
5854  }
5855 }
5857 //*************************************************************************************************
5858 
5859 
5860 //*************************************************************************************************
5872 template< typename Type // Data type of the matrix
5873  , bool AF // Alignment flag
5874  , bool PF // Padding flag
5875  , typename RT > // Result type
5876 template< typename MT > // Type of the right-hand side dense matrix
5877 inline void CustomMatrix<Type,AF,PF,true,RT>::subAssign( const DenseMatrix<MT,false>& rhs )
5878 {
5880 
5881  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5882  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5883 
5884  constexpr size_t block( BLOCK_SIZE );
5885 
5886  for( size_t jj=0UL; jj<n_; jj+=block ) {
5887  const size_t jend( min( n_, jj+block ) );
5888  for( size_t ii=0UL; ii<m_; ii+=block )
5889  {
5890  if( IsLower_v<MT> && ii < jj ) continue;
5891  if( IsUpper_v<MT> && ii > jj ) break;
5892 
5893  for( size_t j=jj; j<jend; ++j )
5894  {
5895  const size_t ibegin( ( IsLower_v<MT> )
5896  ?( max( ( IsStrictlyLower_v<MT> ? j+1UL : j ), ii ) )
5897  :( ii ) );
5898  const size_t iend ( ( IsUpper_v<MT> )
5899  ?( min( ( IsStrictlyUpper_v<MT> ? j : j+1UL ), m_, ii+block ) )
5900  :( min( m_, ii+block ) ) );
5901  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
5902 
5903  for( size_t i=ibegin; i<iend; ++i ) {
5904  v_[i+j*mm_] -= (~rhs)(i,j);
5905  }
5906  }
5907  }
5908  }
5909 }
5911 //*************************************************************************************************
5912 
5913 
5914 //*************************************************************************************************
5926 template< typename Type // Data type of the matrix
5927  , bool AF // Alignment flag
5928  , bool PF // Padding flag
5929  , typename RT > // Result type
5930 template< typename MT > // Type of the right-hand side sparse matrix
5931 inline void CustomMatrix<Type,AF,PF,true,RT>::subAssign( const SparseMatrix<MT,true>& rhs )
5932 {
5933  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5934  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5935 
5936  for( size_t j=0UL; j<(~rhs).columns(); ++j )
5937  for( auto element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
5938  v_[element->index()+j*mm_] -= element->value();
5939 }
5941 //*************************************************************************************************
5942 
5943 
5944 //*************************************************************************************************
5956 template< typename Type // Data type of the matrix
5957  , bool AF // Alignment flag
5958  , bool PF // Padding flag
5959  , typename RT > // Result type
5960 template< typename MT > // Type of the right-hand side sparse matrix
5961 inline void CustomMatrix<Type,AF,PF,true,RT>::subAssign( const SparseMatrix<MT,false>& rhs )
5962 {
5964 
5965  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5966  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5967 
5968  for( size_t i=0UL; i<(~rhs).rows(); ++i )
5969  for( auto element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
5970  v_[i+element->index()*mm_] -= element->value();
5971 }
5973 //*************************************************************************************************
5974 
5975 
5976 //*************************************************************************************************
5988 template< typename Type // Data type of the matrix
5989  , bool AF // Alignment flag
5990  , bool PF // Padding flag
5991  , typename RT > // Result type
5992 template< typename MT > // Type of the right-hand side dense matrix
5993 inline auto CustomMatrix<Type,AF,PF,true,RT>::schurAssign( const DenseMatrix<MT,true>& rhs )
5994  -> DisableIf_t< VectorizedSchurAssign_v<MT> >
5995 {
5996  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
5997  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
5998 
5999  const size_t ipos( m_ & size_t(-2) );
6000  BLAZE_INTERNAL_ASSERT( ( m_ - ( m_ % 2UL ) ) == ipos, "Invalid end calculation" );
6001 
6002  for( size_t j=0UL; j<n_; ++j ) {
6003  for( size_t i=0UL; i<ipos; i+=2UL ) {
6004  v_[i +j*mm_] *= (~rhs)(i ,j);
6005  v_[i+1UL+j*mm_] *= (~rhs)(i+1UL,j);
6006  }
6007  if( ipos < m_ ) {
6008  v_[ipos+j*mm_] *= (~rhs)(ipos,j);
6009  }
6010  }
6011 }
6013 //*************************************************************************************************
6014 
6015 
6016 //*************************************************************************************************
6029 template< typename Type // Data type of the matrix
6030  , bool AF // Alignment flag
6031  , bool PF // Padding flag
6032  , typename RT > // Result type
6033 template< typename MT > // Type of the right-hand side dense matrix
6034 inline auto CustomMatrix<Type,AF,PF,true,RT>::schurAssign( const DenseMatrix<MT,true>& rhs )
6035  -> EnableIf_t< VectorizedSchurAssign_v<MT> >
6036 {
6038 
6039  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
6040  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
6041 
6042  constexpr bool remainder( !PF || !IsPadded_v<MT> );
6043 
6044  for( size_t j=0UL; j<n_; ++j )
6045  {
6046  const size_t ipos( ( remainder )?( m_ & size_t(-SIMDSIZE) ):( m_ ) );
6047  BLAZE_INTERNAL_ASSERT( !remainder || ( m_ - ( m_ % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
6048 
6049  size_t i( 0UL );
6050  Iterator left( begin(j) );
6051  ConstIterator_t<MT> right( (~rhs).begin(j) );
6052 
6053  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6054  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6055  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6056  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6057  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6058  }
6059  for( ; i<ipos; i+=SIMDSIZE ) {
6060  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6061  }
6062  for( ; remainder && i<m_; ++i ) {
6063  *left *= *right; ++left; ++right;
6064  }
6065  }
6066 }
6068 //*************************************************************************************************
6069 
6070 
6071 //*************************************************************************************************
6083 template< typename Type // Data type of the matrix
6084  , bool AF // Alignment flag
6085  , bool PF // Padding flag
6086  , typename RT > // Result type
6087 template< typename MT > // Type of the right-hand side dense matrix
6088 inline void CustomMatrix<Type,AF,PF,true,RT>::schurAssign( const DenseMatrix<MT,false>& rhs )
6089 {
6091 
6092  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
6093  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
6094 
6095  constexpr size_t block( BLOCK_SIZE );
6096 
6097  for( size_t jj=0UL; jj<n_; jj+=block ) {
6098  const size_t jend( min( n_, jj+block ) );
6099  for( size_t ii=0UL; ii<m_; ii+=block ) {
6100  const size_t iend( min( m_, ii+block ) );
6101  for( size_t j=jj; j<jend; ++j ) {
6102  for( size_t i=ii; i<iend; ++i ) {
6103  v_[i+j*mm_] *= (~rhs)(i,j);
6104  }
6105  }
6106  }
6107  }
6108 }
6110 //*************************************************************************************************
6111 
6112 
6113 //*************************************************************************************************
6125 template< typename Type // Data type of the matrix
6126  , bool AF // Alignment flag
6127  , bool PF // Padding flag
6128  , typename RT > // Result type
6129 template< typename MT > // Type of the right-hand side sparse matrix
6130 inline void CustomMatrix<Type,AF,PF,true,RT>::schurAssign( const SparseMatrix<MT,true>& rhs )
6131 {
6132  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
6133  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
6134 
6135  const ResultType tmp( serial( *this ) );
6136 
6137  reset();
6138 
6139  for( size_t j=0UL; j<(~rhs).columns(); ++j )
6140  for( auto element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
6141  v_[element->index()+j*mm_] = tmp(element->index(),j) * element->value();
6142 }
6144 //*************************************************************************************************
6145 
6146 
6147 //*************************************************************************************************
6159 template< typename Type // Data type of the matrix
6160  , bool AF // Alignment flag
6161  , bool PF // Padding flag
6162  , typename RT > // Result type
6163 template< typename MT > // Type of the right-hand side sparse matrix
6164 inline void CustomMatrix<Type,AF,PF,true,RT>::schurAssign( const SparseMatrix<MT,false>& rhs )
6165 {
6167 
6168  BLAZE_INTERNAL_ASSERT( m_ == (~rhs).rows() , "Invalid number of rows" );
6169  BLAZE_INTERNAL_ASSERT( n_ == (~rhs).columns(), "Invalid number of columns" );
6170 
6171  const ResultType tmp( serial( *this ) );
6172 
6173  reset();
6174 
6175  for( size_t i=0UL; i<(~rhs).rows(); ++i )
6176  for( auto element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
6177  v_[i+element->index()*mm_] = tmp(i,element->index()) * element->value();
6178 }
6180 //*************************************************************************************************
6181 
6182 
6183 
6184 
6185 
6186 
6187 
6188 
6189 //=================================================================================================
6190 //
6191 // CUSTOMMATRIX OPERATORS
6192 //
6193 //=================================================================================================
6194 
6195 //*************************************************************************************************
6198 template< typename Type, bool AF, bool PF, bool SO, typename RT >
6199 void reset( CustomMatrix<Type,AF,PF,SO,RT>& m );
6200 
6201 template< typename Type, bool AF, bool PF, bool SO, typename RT >
6202 void reset( CustomMatrix<Type,AF,PF,SO,RT>& m, size_t i );
6203 
6204 template< typename Type, bool AF, bool PF, bool SO, typename RT >
6205 void clear( CustomMatrix<Type,AF,PF,SO,RT>& m );
6206 
6207 template< bool RF, typename Type, bool AF, bool PF, bool SO, typename RT >
6208 bool isDefault( const CustomMatrix<Type,AF,PF,SO,RT>& m );
6209 
6210 template< typename Type, bool AF, bool PF, bool SO, typename RT >
6211 bool isIntact( const CustomMatrix<Type,AF,PF,SO,RT>& m );
6212 
6213 template< typename Type, bool AF, bool PF, bool SO, typename RT >
6214 void swap( CustomMatrix<Type,AF,PF,SO,RT>& a, CustomMatrix<Type,AF,PF,SO,RT>& b ) noexcept;
6216 //*************************************************************************************************
6217 
6218 
6219 //*************************************************************************************************
6226 template< typename Type // Data type of the matrix
6227  , bool AF // Alignment flag
6228  , bool PF // Padding flag
6229  , bool SO // Storage order
6230  , typename RT > // Result type
6232 {
6233  m.reset();
6234 }
6235 //*************************************************************************************************
6236 
6237 
6238 //*************************************************************************************************
6251 template< typename Type // Data type of the matrix
6252  , bool AF // Alignment flag
6253  , bool PF // Padding flag
6254  , bool SO // Storage order
6255  , typename RT > // Result type
6256 inline void reset( CustomMatrix<Type,AF,PF,SO,RT>& m, size_t i )
6257 {
6258  m.reset( i );
6259 }
6260 //*************************************************************************************************
6261 
6262 
6263 //*************************************************************************************************
6270 template< typename Type // Data type of the matrix
6271  , bool AF // Alignment flag
6272  , bool PF // Padding flag
6273  , bool SO // Storage order
6274  , typename RT > // Result type
6276 {
6277  m.clear();
6278 }
6279 //*************************************************************************************************
6280 
6281 
6282 //*************************************************************************************************
6310 template< bool RF // Relaxation flag
6311  , typename Type // Data type of the matrix
6312  , bool AF // Alignment flag
6313  , bool PF // Padding flag
6314  , bool SO // Storage order
6315  , typename RT > // Result type
6317 {
6318  return ( m.rows() == 0UL && m.columns() == 0UL );
6319 }
6320 //*************************************************************************************************
6321 
6322 
6323 //*************************************************************************************************
6344 template< typename Type // Data type of the matrix
6345  , bool AF // Alignment flag
6346  , bool PF // Padding flag
6347  , bool SO // Storage order
6348  , typename RT > // Result type
6350 {
6351  return ( m.rows() * m.columns() <= m.capacity() );
6352 }
6353 //*************************************************************************************************
6354 
6355 
6356 //*************************************************************************************************
6364 template< typename Type // Data type of the matrix
6365  , bool AF // Alignment flag
6366  , bool PF // Padding flag
6367  , bool SO // Storage order
6368  , typename RT > // Result type
6370 {
6371  a.swap( b );
6372 }
6373 //*************************************************************************************************
6374 
6375 
6376 
6377 
6378 //=================================================================================================
6379 //
6380 // HASCONSTDATAACCESS SPECIALIZATIONS
6381 //
6382 //=================================================================================================
6383 
6384 //*************************************************************************************************
6386 template< typename T, bool AF, bool PF, bool SO, typename RT >
6387 struct HasConstDataAccess< CustomMatrix<T,AF,PF,SO,RT> >
6388  : public TrueType
6389 {};
6391 //*************************************************************************************************
6392 
6393 
6394 
6395 
6396 //=================================================================================================
6397 //
6398 // HASMUTABLEDATAACCESS SPECIALIZATIONS
6399 //
6400 //=================================================================================================
6401 
6402 //*************************************************************************************************
6404 template< typename T, bool AF, bool PF, bool SO, typename RT >
6405 struct HasMutableDataAccess< CustomMatrix<T,AF,PF,SO,RT> >
6406  : public TrueType
6407 {};
6409 //*************************************************************************************************
6410 
6411 
6412 
6413 
6414 //=================================================================================================
6415 //
6416 // ISCUSTOM SPECIALIZATIONS
6417 //
6418 //=================================================================================================
6419 
6420 //*************************************************************************************************
6422 template< typename T, bool AF, bool PF, bool SO, typename RT >
6423 struct IsCustom< CustomMatrix<T,AF,PF,SO,RT> >
6424  : public TrueType
6425 {};
6427 //*************************************************************************************************
6428 
6429 
6430 
6431 
6432 //=================================================================================================
6433 //
6434 // ISALIGNED SPECIALIZATIONS
6435 //
6436 //=================================================================================================
6437 
6438 //*************************************************************************************************
6440 template< typename T, bool PF, bool SO, typename RT >
6441 struct IsAligned< CustomMatrix<T,aligned,PF,SO,RT> >
6442  : public TrueType
6443 {};
6445 //*************************************************************************************************
6446 
6447 
6448 
6449 
6450 //=================================================================================================
6451 //
6452 // ISCONTIGUOUS SPECIALIZATIONS
6453 //
6454 //=================================================================================================
6455 
6456 //*************************************************************************************************
6458 template< typename T, bool AF, bool PF, bool SO, typename RT >
6459 struct IsContiguous< CustomMatrix<T,AF,PF,SO,RT> >
6460  : public TrueType
6461 {};
6463 //*************************************************************************************************
6464 
6465 
6466 
6467 
6468 
6469 //=================================================================================================
6470 //
6471 // ISPADDED SPECIALIZATIONS
6472 //
6473 //=================================================================================================
6474 
6475 //*************************************************************************************************
6477 template< typename T, bool AF, bool SO, typename RT >
6478 struct IsPadded< CustomMatrix<T,AF,padded,SO,RT> >
6479  : public TrueType
6480 {};
6482 //*************************************************************************************************
6483 
6484 } // namespace blaze
6485 
6486 #endif
Constraint on the data type.
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Header file for auxiliary alias declarations.
Headerfile for the generic min algorithm.
Header file for kernel specific block sizes.
CustomMatrix()
The default constructor for CustomMatrix.
Definition: CustomMatrix.h:729
void swap(CustomMatrix &m) noexcept
Swapping the contents of two matrices.
Definition: CustomMatrix.h:1851
BLAZE_ALWAYS_INLINE void storea(size_t i, size_t j, const SIMDType &value) noexcept
Aligned store of a SIMD element of the matrix.
Definition: CustomMatrix.h:2308
typename T1::template Rebind< T2 >::Other Rebind_t
Alias declaration for nested Rebind class templates.The Rebind_t alias declaration provides a conveni...
Definition: Aliases.h:310
Header file for the nextMultiple shim.
#define BLAZE_USER_ASSERT(expr, msg)
Run time assertion macro for user checks.In case of an invalid run time expression,...
Definition: Assert.h:117
Reference at(size_t i, size_t j)
Checked access to the matrix elements.
Definition: CustomMatrix.h:964
Header file for the alignment flag values.
bool isAligned() const noexcept
Returns whether the matrix is properly aligned in memory.
Definition: CustomMatrix.h:2120
Header file for basic type definitions.
void swap(CustomMatrix< Type, AF, PF, SO, RT > &a, CustomMatrix< Type, AF, PF, SO, RT > &b) noexcept
Swapping the contents of two custom matrices.
Definition: CustomMatrix.h:6369
typename If< Condition, T1, T2 >::Type If_t
Auxiliary alias template for the If class template.The If_t alias template provides a convenient shor...
Definition: If.h:109
typename T::ResultType ResultType_t
Alias declaration for nested ResultType type definitions.The ResultType_t alias declaration provides ...
Definition: Aliases.h:390
constexpr bool IsSIMDCombinable_v
Auxiliary variable template for the IsSIMDCombinable type trait.The IsSIMDCombinable_v variable templ...
Definition: IsSIMDCombinable.h:137
CustomMatrix< Type, AF, PF, SO, RT > This
Type of this CustomMatrix instance.
Definition: CustomMatrix.h:411
Header file for the IsSparseMatrix type trait.
size_t spacing() const noexcept
Returns the spacing between the beginning of two rows/columns.
Definition: CustomMatrix.h:1665
Header file for the serial shim.
Header file for the IsDiagonal type trait.
Header file for the IsSame and IsStrictlySame type traits.
BLAZE_ALWAYS_INLINE const EnableIf_t< IsIntegral_v< T > &&HasSize_v< T, 1UL >, If_t< IsSigned_v< T >, SIMDint8, SIMDuint8 > > loadu(const T *address) noexcept
Loads a vector of 1-byte integral values.
Definition: Loadu.h:76
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:372
Generic wrapper for the clear() function.
Definition: Clear.h:60
Rebind_t< RT, RemoveConst_t< NewType > > RRT
The rebound result type.
Definition: CustomMatrix.h:442
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:595
Header file for the IsIntegral type trait.
typename SIMDTrait< T >::Type SIMDTrait_t
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_t alias declaration provid...
Definition: SIMDTrait.h:315
void ctranspose(Matrix< MT, SO > &matrix)
In-place conjugate transpose of the given matrix.
Definition: Matrix.h:851
Header file for the MAYBE_UNUSED function template.
Iterator begin(size_t i) noexcept
Returns an iterator to the first element of row/column i.
Definition: CustomMatrix.h:1118
#define BLAZE_CONSTRAINT_MUST_NOT_BE_VOLATILE(T)
Constraint on the data type.In case the given data type is a volatile-qualified type,...
Definition: Volatile.h:79
size_t n_
The current number of columns of the matrix.
Definition: CustomMatrix.h:687
BLAZE_ALWAYS_INLINE void cswap(T &a, T &b) noexcept(IsNumeric_v< T >)
Swapping two conjugated values/objects.
Definition: Conjugate.h:195
constexpr bool HasSIMDAdd_v
Auxiliary variable template for the HasSIMDAdd type trait.The HasSIMDAdd_v variable template provides...
Definition: HasSIMDAdd.h:187
BoolConstant< true > TrueType
Type traits base class.The TrueType class is used as base class for type traits and value traits that...
Definition: IntegralConstant.h:132
Header file for all forward declarations of the math module.
auto schurAssign(const DenseMatrix< MT, SO > &rhs) -> DisableIf_t< VectorizedSchurAssign_v< MT > >
Default implementation of the Schur product assignment of a row-major dense matrix.
Definition: CustomMatrix.h:3090
BLAZE_ALWAYS_INLINE EnableIf_t< IsIntegral_v< T1 > &&HasSize_v< T1, 1UL > > storeu(T1 *address, const SIMDi8< T2 > &value) noexcept
Unaligned store of a vector of 1-byte integral values.
Definition: Storeu.h:75
Header file for the extended initializer_list functionality.
System settings for performance optimizations.
Header file for the Clear functor.
DenseIterator< Type, AF > Iterator
Iterator over non-constant elements.
Definition: CustomMatrix.h:433
Iterator end(size_t i) noexcept
Returns an iterator just past the last element of row/column i.
Definition: CustomMatrix.h:1193
constexpr size_t columns(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of columns of the matrix.
Definition: Matrix.h:514
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: CustomMatrix.h:1626
typename T::template Resize< Ns... >::Other Resize_t
Alias declaration for nested Resize class templates.The Resize_t alias declaration provides a conveni...
Definition: Aliases.h:370
CustomMatrix & operator=(const Type &set)
Homogenous assignment to all matrix elements.
Definition: CustomMatrix.h:1271
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes....
Definition: DenseMatrix.h:81
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes....
Definition: Forward.h:145
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.The ElementType_t alias declaration provide...
Definition: Aliases.h:170
Type * v_
The custom array of elements.
Definition: CustomMatrix.h:689
static constexpr bool simdEnabled
Compilation flag for SIMD optimization.
Definition: CustomMatrix.h:464
Constraint on the data type.
constexpr bool unpadded
Padding flag for unpadded vectors and matrices.Via this flag it is possible to specify custom vectors...
Definition: PaddingFlag.h:64
BLAZE_ALWAYS_INLINE SIMDType load(size_t i, size_t j) const noexcept
Load of a SIMD element of the matrix.
Definition: CustomMatrix.h:2170
Header file for the SparseMatrix base class.
Type * Pointer
Pointer to a non-constant matrix value.
Definition: CustomMatrix.h:430
typename EnableIf< Condition, T >::Type EnableIf_t
Auxiliary type for the EnableIf class template.The EnableIf_t alias declaration provides a convenient...
Definition: EnableIf.h:138
Headerfile for the generic max algorithm.
Header file for the DisableIf class template.
auto subAssign(const DenseMatrix< MT, SO > &rhs) -> DisableIf_t< VectorizedSubAssign_v< MT > >
Default implementation of the subtraction assignment of a row-major dense matrix.
Definition: CustomMatrix.h:2861
CustomMatrix & transpose()
In-place transpose of the matrix.
Definition: CustomMatrix.h:1884
Header file for the IsCustom type trait.
Header file for the IsStrictlyUpper type trait.
SIMDTrait_t< ElementType > SIMDType
SIMD type of the matrix elements.
Definition: CustomMatrix.h:424
Namespace of the Blaze C++ math library.
Definition: Blaze.h:58
#define BLAZE_ALWAYS_INLINE
Platform dependent setup of an enforced inline keyword.
Definition: Inline.h:85
decltype(auto) ctrans(const DenseMatrix< MT, SO > &dm)
Returns the conjugate transpose matrix of dm.
Definition: DMatMapExpr.h:1361
size_t m_
The current number of rows of the matrix.
Definition: CustomMatrix.h:686
Header file for the If class template.
ConstIterator cbegin(size_t i) const noexcept
Returns an iterator to the first element of row/column i.
Definition: CustomMatrix.h:1168
size_t capacity() const noexcept
Returns the maximum capacity of the matrix.
Definition: CustomMatrix.h:1682
#define BLAZE_CONSTRAINT_MUST_NOT_BE_POINTER_TYPE(T)
Constraint on the data type.In case the given data type T is not a pointer type, a compilation error ...
Definition: Pointer.h:79
Header file for the IsSMPAssignable type trait.
CustomMatrix & ctranspose()
In-place conjugate transpose of the matrix.
Definition: CustomMatrix.h:1914
constexpr bool HasSIMDSub_v
Auxiliary variable template for the HasSIMDSub type trait.The HasSIMDSub_v variable template provides...
Definition: HasSIMDSub.h:187
decltype(auto) min(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise minimum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1162
Resize_t< RT, NewM, NewN > RRT
The resized result type.
Definition: CustomMatrix.h:453
CustomMatrix< NewType, AF, PF, SO, RRT > Other
The type of the other CustomMatrix.
Definition: CustomMatrix.h:443
const This & CompositeType
Data type for composite expression templates.
Definition: CustomMatrix.h:426
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
Header file for the DenseMatrix base class.
Header file for the DenseIterator class template.
decltype(auto) inv(const DenseMatrix< MT, SO > &dm)
Calculation of the inverse of the given dense matrix.
Definition: DMatInvExpr.h:423
RT ResultType
Result type for expression template evaluations.
Definition: CustomMatrix.h:415
Header file for all SIMD functionality.
DenseIterator< const Type, AF > ConstIterator
Iterator over constant elements.
Definition: CustomMatrix.h:434
BLAZE_ALWAYS_INLINE SIMDType loadu(size_t i, size_t j) const noexcept
Unaligned load of a SIMD element of the matrix.
Definition: CustomMatrix.h:2239
BLAZE_ALWAYS_INLINE void storeu(size_t i, size_t j, const SIMDType &value) noexcept
Unaligned store of a SIMD element of the matrix.
Definition: CustomMatrix.h:2347
Constraint on the data type.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_DIAGONAL_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a diagonal matrix type,...
Definition: Diagonal.h:79
Header file for the IsLower type trait.
constexpr void MAYBE_UNUSED(const Args &...)
Suppression of unused parameter warnings.
Definition: MaybeUnused.h:81
Header file for the IsAligned type trait.
BLAZE_ALWAYS_INLINE EnableIf_t< IsIntegral_v< T1 > &&HasSize_v< T1, 1UL > > storea(T1 *address, const SIMDi8< T2 > &value) noexcept
Aligned store of a vector of 1-byte integral values.
Definition: Storea.h:78
Header file for the default storage order for all vectors of the Blaze library.
Generic wrapper for the null function.
Definition: Noop.h:60
#define BLAZE_CONSTRAINT_MUST_BE_VECTORIZABLE_TYPE(T)
Constraint on the data type.In case the given data type T is not a vectorizable data type,...
Definition: Vectorizable.h:61
Reference operator()(size_t i, size_t j) noexcept
2D-access to the matrix elements.
Definition: CustomMatrix.h:913
BLAZE_ALWAYS_INLINE void store(size_t i, size_t j, const SIMDType &value) noexcept
Store of a SIMD element of the matrix.
Definition: CustomMatrix.h:2276
Header file for the misalignment function.
Header file for the exception macros of the math module.
decltype(auto) max(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise maximum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1198
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:438
Rebind mechanism to obtain a CustomMatrix with different data/element type.
Definition: CustomMatrix.h:441
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
void clear(const DiagonalProxy< MT > &proxy)
Clearing the represented element.
Definition: DiagonalProxy.h:615
Header file for the IsPadded type trait.
bool isAliased(const Other *alias) const noexcept
Returns whether the matrix is aliased with the given address alias.
Definition: CustomMatrix.h:2099
const Type * ConstPointer
Pointer to a constant matrix value.
Definition: CustomMatrix.h:431
size_t nonZeros() const
Returns the total number of non-zero elements in the matrix.
Definition: CustomMatrix.h:1724
Header file for the IsVectorizable type trait.
typename T::OppositeType OppositeType_t
Alias declaration for nested OppositeType type definitions.The OppositeType_t alias declaration provi...
Definition: Aliases.h:270
bool canAlias(const Other *alias) const noexcept
Returns whether the matrix can alias with the given address alias.
Definition: CustomMatrix.h:2076
Header file for the conjugate shim.
Flag for aligned vectors and matrices.
Definition: AlignmentFlag.h:65
Header file for the IsNumeric type trait.
Header file for the HasConstDataAccess type trait.
Resize mechanism to obtain a CustomMatrix with different fixed dimensions.
Definition: CustomMatrix.h:452
Header file for the RemoveConst type trait.
const Type & ReturnType
Return type for expression template evaluations.
Definition: CustomMatrix.h:425
Header file for the IsSIMDCombinable type trait.
BLAZE_ALWAYS_INLINE void stream(size_t i, size_t j, const SIMDType &value) noexcept
Aligned, non-temporal store of a SIMD element of the matrix.
Definition: CustomMatrix.h:2385
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type,...
Definition: Symmetric.h:79
Header file for the HasSIMDMult type trait.
Header file for the IsConst type trait.
Type ElementType
Type of the matrix elements.
Definition: CustomMatrix.h:423
typename T::TransposeType TransposeType_t
Alias declaration for nested TransposeType type definitions.The TransposeType_t alias declaration pro...
Definition: Aliases.h:470
BLAZE_ALWAYS_INLINE void conjugate(T &a) noexcept(IsNumeric_v< T >)
In-place conjugation of the given value/object.
Definition: Conjugate.h:120
Header file for run time assertion macros.
static constexpr size_t SIMDSIZE
The number of elements packed within a single SIMD element.
Definition: CustomMatrix.h:618
auto smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:131
auto addAssign(const DenseMatrix< MT, SO > &rhs) -> DisableIf_t< VectorizedAddAssign_v< MT > >
Default implementation of the addition assignment of a row-major dense matrix.
Definition: CustomMatrix.h:2632
Header file for the IsContiguous type trait.
void reset()
Reset to the default initial values.
Definition: CustomMatrix.h:1780
Type & Reference
Reference to a non-constant matrix value.
Definition: CustomMatrix.h:428
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:295
Header file for the cache size of the target architecture.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type,...
Definition: Reference.h:79
size_t nn_
The number of elements between two rows.
Definition: CustomMatrix.h:688
Header file for the isDefault shim.
void swap(DiagonalMatrix< MT, SO, DF > &a, DiagonalMatrix< MT, SO, DF > &b) noexcept
Swapping the contents of two matrices.
Definition: DiagonalMatrix.h:282
BLAZE_ALWAYS_INLINE const EnableIf_t< IsIntegral_v< T > &&HasSize_v< T, 1UL >, If_t< IsSigned_v< T >, SIMDint8, SIMDuint8 > > set(T value) noexcept
Sets all values in the vector to the given 1-byte integral value.
Definition: Set.h:75
constexpr size_t size(const Matrix< MT, SO > &matrix) noexcept
Returns the total number of elements of the matrix.
Definition: Matrix.h:530
Constraint on the data type.
Base class for matrices.The Matrix class is a base class for all dense and sparse matrix classes with...
Definition: Forward.h:114
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
Constraint on the data type.
Constraint on the data type.
Header file for the HasSIMDSub type trait.
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:808
Header file for the HasMutableDataAccess type trait.
Header file for the Noop functor.
BLAZE_ALWAYS_INLINE SIMDType loada(size_t i, size_t j) const noexcept
Aligned load of a SIMD element of the matrix.
Definition: CustomMatrix.h:2201
Pointer data() noexcept
Low-level data access to the matrix elements.
Definition: CustomMatrix.h:1025
auto smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:194
constexpr size_t rows(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of rows of the matrix.
Definition: Matrix.h:498
void clear()
Clearing the matrix.
Definition: CustomMatrix.h:1830
BLAZE_ALWAYS_INLINE const EnableIf_t< IsIntegral_v< T > &&HasSize_v< T, 1UL >, If_t< IsSigned_v< T >, SIMDint8, SIMDuint8 > > loada(const T *address) noexcept
Loads a vector of 1-byte integral values.
Definition: Loada.h:79
Header file for the padding flag values.
Implementation of a generic iterator for dense vectors and matrices.The DenseIterator represents a ge...
Definition: DenseIterator.h:58
auto assign(const DenseMatrix< MT, SO > &rhs) -> DisableIf_t< VectorizedAssign_v< MT > >
Default implementation of the assignment of a row-major dense matrix.
Definition: CustomMatrix.h:2419
typename T::ConstIterator ConstIterator_t
Alias declaration for nested ConstIterator type definitions.The ConstIterator_t alias declaration pro...
Definition: Aliases.h:110
BLAZE_ALWAYS_INLINE EnableIf_t< IsIntegral_v< T1 > &&HasSize_v< T1, 1UL > > stream(T1 *address, const SIMDi8< T2 > &value) noexcept
Aligned, non-temporal store of a vector of 1-byte integral values.
Definition: Stream.h:74
BLAZE_ALWAYS_INLINE bool checkAlignment(const T *address)
Checks the alignment of the given address.
Definition: AlignmentCheck.h:68
decltype(auto) trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:765
CustomMatrix< Type, AF, PF, SO, RRT > Other
The type of the other CustomMatrix.
Definition: CustomMatrix.h:454
const Type & ConstReference
Reference to a constant matrix value.
Definition: CustomMatrix.h:429
auto smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:162
Header file for the default transpose flag for all vectors of the Blaze library.
Initializer list type of the Blaze library.
DenseMatrix< This, SO > BaseType
Base type of this CustomMatrix instance.
Definition: CustomMatrix.h:412
Efficient implementation of a customizable matrix.The CustomMatrix class template provides the functi...
Definition: CustomMatrix.h:406
constexpr bool HasSIMDMult_v
Auxiliary variable template for the HasSIMDMult type trait.The HasSIMDMult_v variable template provid...
Definition: HasSIMDMult.h:188
Header file for the alignment check function.
Header file for the IntegralConstant class template.
OppositeType_t< RT > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: CustomMatrix.h:418
bool isIntact(const DiagonalMatrix< MT, SO, DF > &m)
Returns whether the invariants of the given diagonal matrix are intact.
Definition: DiagonalMatrix.h:264
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: CustomMatrix.h:1643
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:635
#define BLAZE_THROW_LOGIC_ERROR(MESSAGE)
Macro for the emission of a std::logic_error exception.This macro encapsulates the default way of Bla...
Definition: Exception.h:187
Header file for the IsUpper type trait.
typename DisableIf< Condition, T >::Type DisableIf_t
Auxiliary type for the DisableIf class template.The DisableIf_t alias declaration provides a convenie...
Definition: DisableIf.h:138
constexpr bool IsDiagonal_v
Auxiliary variable template for the IsDiagonal type trait.The IsDiagonal_v variable template provides...
Definition: IsDiagonal.h:148
#define BLAZE_STATIC_ASSERT(expr)
Compile time assertion macro.In case of an invalid compile time expression, a compilation error is cr...
Definition: StaticAssert.h:112
ConstIterator cend(size_t i) const noexcept
Returns an iterator just past the last element of row/column i.
Definition: CustomMatrix.h:1243
bool canSMPAssign() const noexcept
Returns whether the matrix can be used in SMP assignments.
Definition: CustomMatrix.h:2142
System settings for the inline keywords.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
constexpr bool padded
Padding flag for padded vectors and matrices.Via this flag it is possible to specify custom vectors a...
Definition: PaddingFlag.h:86
TransposeType_t< RT > TransposeType
Transpose type for expression template evaluations.
Definition: CustomMatrix.h:421
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression,...
Definition: Assert.h:101
constexpr size_t determineColumns(initializer_list< initializer_list< Type > > list) noexcept
Determines the maximum number of columns specified by the given initializer list.
Definition: InitializerList.h:108
Header file for the clear shim.
static constexpr bool smpAssignable
Compilation flag for SMP assignments.
Definition: CustomMatrix.h:470
void transpose(Matrix< MT, SO > &matrix)
In-place transpose of the given matrix.
Definition: Matrix.h:825