TDMatSMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATSMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATSMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
53 #include <blaze/math/Exception.h>
66 #include <blaze/math/shims/Reset.h>
90 #include <blaze/math/views/Check.h>
95 #include <blaze/util/Assert.h>
96 #include <blaze/util/DisableIf.h>
97 #include <blaze/util/EnableIf.h>
100 #include <blaze/util/mpl/If.h>
101 #include <blaze/util/Types.h>
103 #include <blaze/util/Unused.h>
104 
105 
106 namespace blaze {
107 
108 //=================================================================================================
109 //
110 // CLASS TDMATSMATMULTEXPR
111 //
112 //=================================================================================================
113 
114 //*************************************************************************************************
121 template< typename MT1 // Type of the left-hand side dense matrix
122  , typename MT2 // Type of the right-hand side sparse matrix
123  , bool SF // Symmetry flag
124  , bool HF // Hermitian flag
125  , bool LF // Lower flag
126  , bool UF > // Upper flag
127 class TDMatSMatMultExpr
128  : public MatMatMultExpr< DenseMatrix< TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, true > >
129  , private Computation
130 {
131  private:
132  //**Type definitions****************************************************************************
139  //**********************************************************************************************
140 
141  //**********************************************************************************************
143  static constexpr bool evaluateLeft = ( IsComputation_v<MT1> || RequiresEvaluation_v<MT1> );
144  //**********************************************************************************************
145 
146  //**********************************************************************************************
148  static constexpr bool evaluateRight = ( IsComputation_v<MT2> || RequiresEvaluation_v<MT2> );
149  //**********************************************************************************************
150 
151  //**********************************************************************************************
152  static constexpr bool SYM = ( SF && !( HF || LF || UF ) );
153  static constexpr bool HERM = ( HF && !( LF || UF ) );
154  static constexpr bool LOW = ( LF || ( ( SF || HF ) && UF ) );
155  static constexpr bool UPP = ( UF || ( ( SF || HF ) && LF ) );
156  //**********************************************************************************************
157 
158  //**********************************************************************************************
160 
165  template< typename T1, typename T2, typename T3 >
166  static constexpr bool CanExploitSymmetry_v = IsSymmetric_v<T3>;
168  //**********************************************************************************************
169 
170  //**********************************************************************************************
172 
176  template< typename T1, typename T2, typename T3 >
177  static constexpr bool IsEvaluationRequired_v =
178  ( ( evaluateLeft || evaluateRight ) && !CanExploitSymmetry_v<T1,T2,T3> );
180  //**********************************************************************************************
181 
182  //**********************************************************************************************
184 
187  template< typename T1, typename T2, typename T3 >
188  static constexpr bool UseOptimizedKernel_v =
189  ( useOptimizedKernels &&
190  !IsDiagonal_v<T2> &&
191  !IsResizable_v< ElementType_t<T1> > &&
192  !( IsColumnMajorMatrix_v<T1> && IsResizable_v<ET2> ) );
194  //**********************************************************************************************
195 
196  //**********************************************************************************************
198 
201  template< typename T1, typename T2, typename T3 >
202  static constexpr bool UseDefaultKernel_v = !UseOptimizedKernel_v<T1,T2,T3>;
204  //**********************************************************************************************
205 
206  //**********************************************************************************************
208 
211  using ForwardFunctor = If_t< HERM
212  , DeclHerm
213  , If_t< SYM
214  , DeclSym
215  , If_t< LOW
216  , If_t< UPP
217  , DeclDiag
218  , DeclLow >
219  , If_t< UPP
220  , DeclUpp
221  , Noop > > > >;
223  //**********************************************************************************************
224 
225  public:
226  //**Type definitions****************************************************************************
229 
232 
234  using ResultType = typename If_t< HERM
236  , If_t< SYM
238  , If_t< LOW
239  , If_t< UPP
242  , If_t< UPP
244  , MultTrait<RT1,RT2> > > > >::Type;
245 
249  using ReturnType = const ElementType;
250  using CompositeType = const ResultType;
251 
253  using LeftOperand = If_t< IsExpression_v<MT1>, const MT1, const MT1& >;
254 
256  using RightOperand = If_t< IsExpression_v<MT2>, const MT2, const MT2& >;
257 
260 
263  //**********************************************************************************************
264 
265  //**Compilation flags***************************************************************************
267  static constexpr bool simdEnabled = false;
268 
270  static constexpr bool smpAssignable =
272  //**********************************************************************************************
273 
274  //**Constructor*********************************************************************************
280  explicit inline TDMatSMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
281  : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
282  , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
283  {
284  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
285  }
286  //**********************************************************************************************
287 
288  //**Access operator*****************************************************************************
295  inline ReturnType operator()( size_t i, size_t j ) const {
296  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
297  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
298 
299  if( IsDiagonal_v<MT1> ) {
300  return lhs_(i,i) * rhs_(i,j);
301  }
302  else if( IsDiagonal_v<MT2> ) {
303  return lhs_(i,j) * rhs_(j,j);
304  }
305  else if( IsTriangular_v<MT1> || IsTriangular_v<MT2> ) {
306  const size_t begin( ( IsUpper_v<MT1> )
307  ?( ( IsLower_v<MT2> )
308  ?( max( ( IsStrictlyUpper_v<MT1> ? i+1UL : i )
309  , ( IsStrictlyLower_v<MT2> ? j+1UL : j ) ) )
310  :( IsStrictlyUpper_v<MT1> ? i+1UL : i ) )
311  :( ( IsLower_v<MT2> )
312  ?( IsStrictlyLower_v<MT2> ? j+1UL : j )
313  :( 0UL ) ) );
314  const size_t end( ( IsLower_v<MT1> )
315  ?( ( IsUpper_v<MT2> )
316  ?( min( ( IsStrictlyLower_v<MT1> ? i : i+1UL )
317  , ( IsStrictlyUpper_v<MT2> ? j : j+1UL ) ) )
318  :( IsStrictlyLower_v<MT1> ? i : i+1UL ) )
319  :( ( IsUpper_v<MT2> )
320  ?( IsStrictlyUpper_v<MT2> ? j : j+1UL )
321  :( lhs_.columns() ) ) );
322 
323  if( begin >= end ) return ElementType();
324 
325  const size_t n( end - begin );
326 
327  return subvector( row( lhs_, i, unchecked ), begin, n, unchecked ) *
328  subvector( column( rhs_, j, unchecked ), begin, n, unchecked );
329  }
330  else {
331  return row( lhs_, i, unchecked ) * column( rhs_, j, unchecked );
332  }
333  }
334  //**********************************************************************************************
335 
336  //**At function*********************************************************************************
344  inline ReturnType at( size_t i, size_t j ) const {
345  if( i >= lhs_.rows() ) {
346  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
347  }
348  if( j >= rhs_.columns() ) {
349  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
350  }
351  return (*this)(i,j);
352  }
353  //**********************************************************************************************
354 
355  //**Rows function*******************************************************************************
360  inline size_t rows() const noexcept {
361  return lhs_.rows();
362  }
363  //**********************************************************************************************
364 
365  //**Columns function****************************************************************************
370  inline size_t columns() const noexcept {
371  return rhs_.columns();
372  }
373  //**********************************************************************************************
374 
375  //**Left operand access*************************************************************************
380  inline LeftOperand leftOperand() const noexcept {
381  return lhs_;
382  }
383  //**********************************************************************************************
384 
385  //**Right operand access************************************************************************
390  inline RightOperand rightOperand() const noexcept {
391  return rhs_;
392  }
393  //**********************************************************************************************
394 
395  //**********************************************************************************************
401  template< typename T >
402  inline bool canAlias( const T* alias ) const noexcept {
403  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
404  }
405  //**********************************************************************************************
406 
407  //**********************************************************************************************
413  template< typename T >
414  inline bool isAliased( const T* alias ) const {
415  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
416  }
417  //**********************************************************************************************
418 
419  //**********************************************************************************************
424  inline bool isAligned() const noexcept {
425  return lhs_.isAligned();
426  }
427  //**********************************************************************************************
428 
429  //**********************************************************************************************
434  inline bool canSMPAssign() const noexcept {
435  return ( rows() * columns() >= SMP_TDMATSMATMULT_THRESHOLD ) && !IsDiagonal_v<MT1>;
436  }
437  //**********************************************************************************************
438 
439  private:
440  //**Member variables****************************************************************************
443  //**********************************************************************************************
444 
445  //**Assignment to dense matrices****************************************************************
458  template< typename MT // Type of the target dense matrix
459  , bool SO > // Storage order of the target dense matrix
460  friend inline auto assign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
462  {
464 
465  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
466  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
467 
468  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
469  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
470 
471  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
472  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
473  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
474  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
475  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
476  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
477 
478  TDMatSMatMultExpr::selectAssignKernel( ~lhs, A, B );
479  }
481  //**********************************************************************************************
482 
483  //**Assignment to dense matrices (kernel selection)*********************************************
494  template< typename MT3 // Type of the left-hand side target matrix
495  , typename MT4 // Type of the left-hand side matrix operand
496  , typename MT5 > // Type of the right-hand side matrix operand
497  static inline void selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
498  {
499  if( C.rows() * C.columns() < TDMATSMATMULT_THRESHOLD )
500  selectSmallAssignKernel( C, A, B );
501  else
502  selectLargeAssignKernel( C, A, B );
503  }
505  //**********************************************************************************************
506 
507  //**Default assignment to dense matrices********************************************************
522  template< typename MT3 // Type of the left-hand side target matrix
523  , typename MT4 // Type of the left-hand side matrix operand
524  , typename MT5 > // Type of the right-hand side matrix operand
525  static inline void selectDefaultAssignKernel( MT3& C, const MT4& A, const MT5& B )
526  {
527  reset( C );
528 
529  for( size_t j=0UL; j<B.rows(); ++j )
530  {
531  auto element( B.begin(j) );
532  const auto end( B.end(j) );
533 
534  if( IsDiagonal_v<MT4> )
535  {
536  for( ; element!=end; ++element ) {
537  C(j,element->index()) = A(j,j) * element->value();
538  }
539  }
540  else
541  {
542  for( ; element!=end; ++element )
543  {
544  const size_t j1( element->index() );
545 
546  const size_t ibegin( ( IsLower_v<MT4> )
547  ?( ( IsStrictlyLower_v<MT4> )
548  ?( SYM || HERM || LOW ? max(j1,j+1UL) : j+1UL )
549  :( SYM || HERM || LOW ? max(j1,j) : j ) )
550  :( SYM || HERM || LOW ? j1 : 0UL ) );
551  const size_t iend( ( IsUpper_v<MT4> )
552  ?( ( IsStrictlyUpper_v<MT4> )
553  ?( UPP ? min(j1+1UL,j) : j )
554  :( UPP ? min(j1,j)+1UL : j+1UL ) )
555  :( UPP ? j1+1UL : A.rows() ) );
556 
557  if( ( SYM || HERM || LOW || UPP ) && ( ibegin >= iend ) ) continue;
558  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
559 
560  for( size_t i=ibegin; i<iend; ++i ) {
561  if( isDefault( C(i,j1) ) )
562  C(i,j1) = A(i,j) * element->value();
563  else
564  C(i,j1) += A(i,j) * element->value();
565  }
566  }
567  }
568  }
569 
570  if( SYM || HERM ) {
571  for( size_t j=1UL; j<B.columns(); ++j ) {
572  for( size_t i=0UL; i<j; ++i ) {
573  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
574  }
575  }
576  }
577  }
579  //**********************************************************************************************
580 
581  //**Default assignment to dense matrices (small matrices)***************************************
595  template< typename MT3 // Type of the left-hand side target matrix
596  , typename MT4 // Type of the left-hand side matrix operand
597  , typename MT5 > // Type of the right-hand side matrix operand
598  static inline auto selectSmallAssignKernel( MT3& C, const MT4& A, const MT5& B )
599  -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
600  {
601  selectDefaultAssignKernel( C, A, B );
602  }
604  //**********************************************************************************************
605 
606  //**Optimized assignment to dense matrices (small matrices)*************************************
621  template< typename MT3 // Type of the left-hand side target matrix
622  , typename MT4 // Type of the left-hand side matrix operand
623  , typename MT5 > // Type of the right-hand side matrix operand
624  static inline auto selectSmallAssignKernel( MT3& C, const MT4& A, const MT5& B )
625  -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
626  {
627  reset( C );
628 
629  for( size_t j=0UL; j<B.rows(); ++j )
630  {
631  auto element( B.begin(j) );
632  const auto end( B.end(j) );
633 
634  const size_t nonzeros( B.nonZeros(j) );
635  const size_t kpos( nonzeros & size_t(-4) );
636  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
637 
638  for( size_t k=0UL; k<kpos; k+=4UL )
639  {
640  const size_t j1( element->index() );
641  const ET2 v1( element->value() );
642  ++element;
643  const size_t j2( element->index() );
644  const ET2 v2( element->value() );
645  ++element;
646  const size_t j3( element->index() );
647  const ET2 v3( element->value() );
648  ++element;
649  const size_t j4( element->index() );
650  const ET2 v4( element->value() );
651  ++element;
652 
653  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
654 
655  const size_t ibegin( ( IsLower_v<MT4> )
656  ?( ( IsStrictlyLower_v<MT4> )
657  ?( SYM || HERM || LOW ? max(j1,j+1UL) : j+1UL )
658  :( SYM || HERM || LOW ? max(j1,j) : j ) )
659  :( SYM || HERM || LOW ? j1 : 0UL ) );
660  const size_t iend( ( IsUpper_v<MT4> )
661  ?( ( IsStrictlyUpper_v<MT4> )
662  ?( UPP ? min(j4+1UL,j) : j )
663  :( UPP ? min(j4,j)+1UL : j+1UL ) )
664  :( UPP ? j4+1UL : A.rows() ) );
665 
666  if( ( SYM || HERM || LOW || UPP ) && ( ibegin >= iend ) ) continue;
667  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
668 
669  const size_t inum( iend - ibegin );
670  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
671  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
672 
673  size_t i( ibegin );
674 
675  for( ; i<ipos; i+=4UL ) {
676  C(i ,j1) += A(i ,j) * v1;
677  C(i+1UL,j1) += A(i+1UL,j) * v1;
678  C(i+2UL,j1) += A(i+2UL,j) * v1;
679  C(i+3UL,j1) += A(i+3UL,j) * v1;
680  C(i ,j2) += A(i ,j) * v2;
681  C(i+1UL,j2) += A(i+1UL,j) * v2;
682  C(i+2UL,j2) += A(i+2UL,j) * v2;
683  C(i+3UL,j2) += A(i+3UL,j) * v2;
684  C(i ,j3) += A(i ,j) * v3;
685  C(i+1UL,j3) += A(i+1UL,j) * v3;
686  C(i+2UL,j3) += A(i+2UL,j) * v3;
687  C(i+3UL,j3) += A(i+3UL,j) * v3;
688  C(i ,j4) += A(i ,j) * v4;
689  C(i+1UL,j4) += A(i+1UL,j) * v4;
690  C(i+2UL,j4) += A(i+2UL,j) * v4;
691  C(i+3UL,j4) += A(i+3UL,j) * v4;
692  }
693  for( ; i<iend; ++i ) {
694  C(i,j1) += A(i,j) * v1;
695  C(i,j2) += A(i,j) * v2;
696  C(i,j3) += A(i,j) * v3;
697  C(i,j4) += A(i,j) * v4;
698  }
699  }
700 
701  for( ; element!=end; ++element )
702  {
703  const size_t j1( element->index() );
704  const ET2 v1( element->value() );
705 
706  const size_t ibegin( ( IsLower_v<MT4> )
707  ?( ( IsStrictlyLower_v<MT4> )
708  ?( SYM || HERM || LOW ? max(j1,j+1UL) : j+1UL )
709  :( SYM || HERM || LOW ? max(j1,j) : j ) )
710  :( SYM || HERM || LOW ? j1 : 0UL ) );
711  const size_t iend( ( IsUpper_v<MT4> )
712  ?( ( IsStrictlyUpper_v<MT4> )
713  ?( UPP ? min(j1+1UL,j) : j )
714  :( UPP ? min(j1,j)+1UL : j+1UL ) )
715  :( UPP ? j1+1UL : A.rows() ) );
716 
717  if( ( SYM || HERM || LOW || UPP ) && ( ibegin >= iend ) ) continue;
718  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
719 
720  const size_t inum( iend - ibegin );
721  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
722  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
723 
724  size_t i( ibegin );
725 
726  for( ; i<ipos; i+=4UL ) {
727  C(i ,j1) += A(i ,j) * v1;
728  C(i+1UL,j1) += A(i+1UL,j) * v1;
729  C(i+2UL,j1) += A(i+2UL,j) * v1;
730  C(i+3UL,j1) += A(i+3UL,j) * v1;
731  }
732  for( ; i<iend; ++i ) {
733  C(i,j1) += A(i,j) * v1;
734  }
735  }
736  }
737 
738  if( SYM || HERM ) {
739  for( size_t j=1UL; j<B.columns(); ++j ) {
740  for( size_t i=0UL; i<j; ++i ) {
741  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
742  }
743  }
744  }
745  }
747  //**********************************************************************************************
748 
749  //**Default assignment to dense matrices (large matrices)***************************************
763  template< typename MT3 // Type of the left-hand side target matrix
764  , typename MT4 // Type of the left-hand side matrix operand
765  , typename MT5 > // Type of the right-hand side matrix operand
766  static inline auto selectLargeAssignKernel( MT3& C, const MT4& A, const MT5& B )
767  -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
768  {
769  selectDefaultAssignKernel( C, A, B );
770  }
772  //**********************************************************************************************
773 
774  //**Optimized assignment to dense matrices (large matrices)*************************************
789  template< typename MT3 // Type of the left-hand side target matrix
790  , typename MT4 // Type of the left-hand side matrix operand
791  , typename MT5 > // Type of the right-hand side matrix operand
792  static inline auto selectLargeAssignKernel( MT3& C, const MT4& A, const MT5& B )
793  -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
794  {
796 
797  const ForwardFunctor fwd;
798 
799  const OppositeType_t<MT5> tmp( serial( B ) );
800  assign( C, fwd( A * tmp ) );
801  }
803  //**********************************************************************************************
804 
805  //**Assignment to sparse matrices***************************************************************
818  template< typename MT // Type of the target sparse matrix
819  , bool SO > // Storage order of the target sparse matrix
820  friend inline auto assign( SparseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
821  -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
822  {
824 
825  using TmpType = If_t< SO, ResultType, OppositeType >;
826 
833 
834  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
835  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
836 
837  const ForwardFunctor fwd;
838 
839  const TmpType tmp( serial( rhs ) );
840  assign( ~lhs, fwd( tmp ) );
841  }
843  //**********************************************************************************************
844 
845  //**Restructuring assignment********************************************************************
860  template< typename MT // Type of the target matrix
861  , bool SO > // Storage order of the target matrix
862  friend inline auto assign( Matrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
863  -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
864  {
866 
868 
869  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
870  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
871 
872  const ForwardFunctor fwd;
873 
874  assign( ~lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
875  }
877  //**********************************************************************************************
878 
879  //**Addition assignment to dense matrices*******************************************************
892  template< typename MT // Type of the target dense matrix
893  , bool SO > // Storage order of the target dense matrix
894  friend inline auto addAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
895  -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
896  {
898 
899  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
900  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
901 
902  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
903  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
904 
905  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
906  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
907  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
908  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
909  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
910  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
911 
912  TDMatSMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
913  }
915  //**********************************************************************************************
916 
917  //**Addition assignment to dense matrices (kernel selection)************************************
928  template< typename MT3 // Type of the left-hand side target matrix
929  , typename MT4 // Type of the left-hand side matrix operand
930  , typename MT5 > // Type of the right-hand side matrix operand
931  static inline void selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
932  {
933  if( C.rows() * C.columns() < TDMATSMATMULT_THRESHOLD )
934  selectSmallAddAssignKernel( C, A, B );
935  else
936  selectLargeAddAssignKernel( C, A, B );
937  }
939  //**********************************************************************************************
940 
941  //**Default addition assignment to dense matrices***********************************************
956  template< typename MT3 // Type of the left-hand side target matrix
957  , typename MT4 // Type of the left-hand side matrix operand
958  , typename MT5 > // Type of the right-hand side matrix operand
959  static inline void selectDefaultAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
960  {
961  size_t i( 0UL );
962 
963  for( size_t j=0UL; j<B.rows(); ++j )
964  {
965  auto element( B.begin(j) );
966  const auto end( B.end(j) );
967 
968  if( IsDiagonal_v<MT4> )
969  {
970  for( ; element!=end; ++element ) {
971  C(j,element->index()) += A(j,j) * element->value();
972  }
973  }
974  else
975  {
976  for( ; element!=end; ++element )
977  {
978  const size_t j1( element->index() );
979 
980  const size_t ibegin( ( IsLower_v<MT4> )
981  ?( ( IsStrictlyLower_v<MT4> )
982  ?( LOW ? max(j1,j+1UL) : j+1UL )
983  :( LOW ? max(j1,j) : j ) )
984  :( LOW ? j1 : 0UL ) );
985  const size_t iend( ( IsUpper_v<MT4> )
986  ?( ( IsStrictlyUpper_v<MT4> )
987  ?( UPP ? min(j1+1UL,j) : j )
988  :( UPP ? min(j1,j)+1UL : j+1UL ) )
989  :( UPP ? j1+1UL : A.rows() ) );
990 
991  if( ( LOW || UPP ) && ( ibegin >= iend ) ) continue;
992  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
993 
994  const size_t inum( iend - ibegin );
995  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
996  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
997 
998  for( i=ibegin; i<ipos; i+=4UL ) {
999  C(i ,j1) += A(i ,j) * element->value();
1000  C(i+1UL,j1) += A(i+1UL,j) * element->value();
1001  C(i+2UL,j1) += A(i+2UL,j) * element->value();
1002  C(i+3UL,j1) += A(i+3UL,j) * element->value();
1003  }
1004  for( ; i<iend; ++i ) {
1005  C(i,j1) += A(i,j) * element->value();
1006  }
1007  }
1008  }
1009  }
1010  }
1012  //**********************************************************************************************
1013 
1014  //**Default addition assignment to dense matrices (small matrices)******************************
1028  template< typename MT3 // Type of the left-hand side target matrix
1029  , typename MT4 // Type of the left-hand side matrix operand
1030  , typename MT5 > // Type of the right-hand side matrix operand
1031  static inline auto selectSmallAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1032  -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
1033  {
1034  selectDefaultAddAssignKernel( C, A, B );
1035  }
1037  //**********************************************************************************************
1038 
1039  //**Optimized addition assignment to dense matrices (small matrices)****************************
1054  template< typename MT3 // Type of the left-hand side target matrix
1055  , typename MT4 // Type of the left-hand side matrix operand
1056  , typename MT5 > // Type of the right-hand side matrix operand
1057  static inline auto selectSmallAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1058  -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1059  {
1060  for( size_t j=0UL; j<B.rows(); ++j )
1061  {
1062  auto element( B.begin(j) );
1063  const auto end( B.end(j) );
1064 
1065  const size_t nonzeros( B.nonZeros(j) );
1066  const size_t kpos( nonzeros & size_t(-4) );
1067  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1068 
1069  for( size_t k=0UL; k<kpos; k+=4UL )
1070  {
1071  const size_t j1( element->index() );
1072  const ET2 v1( element->value() );
1073  ++element;
1074  const size_t j2( element->index() );
1075  const ET2 v2( element->value() );
1076  ++element;
1077  const size_t j3( element->index() );
1078  const ET2 v3( element->value() );
1079  ++element;
1080  const size_t j4( element->index() );
1081  const ET2 v4( element->value() );
1082  ++element;
1083 
1084  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1085 
1086  const size_t ibegin( ( IsLower_v<MT4> )
1087  ?( ( IsStrictlyLower_v<MT4> )
1088  ?( LOW ? max(j1,j+1UL) : j+1UL )
1089  :( LOW ? max(j1,j) : j ) )
1090  :( LOW ? j1 : 0UL ) );
1091  const size_t iend( ( IsUpper_v<MT4> )
1092  ?( ( IsStrictlyUpper_v<MT4> )
1093  ?( UPP ? min(j4+1UL,j) : j )
1094  :( UPP ? min(j4,j)+1UL : j+1UL ) )
1095  :( UPP ? j4+1UL : A.rows() ) );
1096 
1097  if( ( LOW || UPP ) && ( ibegin >= iend ) ) continue;
1098  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1099 
1100  const size_t inum( iend - ibegin );
1101  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1102  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1103 
1104  size_t i( ibegin );
1105 
1106  for( i=ibegin; i<ipos; i+=4UL ) {
1107  C(i ,j1) += A(i ,j) * v1;
1108  C(i+1UL,j1) += A(i+1UL,j) * v1;
1109  C(i+2UL,j1) += A(i+2UL,j) * v1;
1110  C(i+3UL,j1) += A(i+3UL,j) * v1;
1111  C(i ,j2) += A(i ,j) * v2;
1112  C(i+1UL,j2) += A(i+1UL,j) * v2;
1113  C(i+2UL,j2) += A(i+2UL,j) * v2;
1114  C(i+3UL,j2) += A(i+3UL,j) * v2;
1115  C(i ,j3) += A(i ,j) * v3;
1116  C(i+1UL,j3) += A(i+1UL,j) * v3;
1117  C(i+2UL,j3) += A(i+2UL,j) * v3;
1118  C(i+3UL,j3) += A(i+3UL,j) * v3;
1119  C(i ,j4) += A(i ,j) * v4;
1120  C(i+1UL,j4) += A(i+1UL,j) * v4;
1121  C(i+2UL,j4) += A(i+2UL,j) * v4;
1122  C(i+3UL,j4) += A(i+3UL,j) * v4;
1123  }
1124  for( ; i<iend; ++i ) {
1125  C(i,j1) += A(i,j) * v1;
1126  C(i,j2) += A(i,j) * v2;
1127  C(i,j3) += A(i,j) * v3;
1128  C(i,j4) += A(i,j) * v4;
1129  }
1130  }
1131 
1132  for( ; element!=end; ++element )
1133  {
1134  const size_t j1( element->index() );
1135  const ET2 v1( element->value() );
1136 
1137  const size_t ibegin( ( IsLower_v<MT4> )
1138  ?( ( IsStrictlyLower_v<MT4> )
1139  ?( LOW ? max(j1,j+1UL) : j+1UL )
1140  :( LOW ? max(j1,j) : j ) )
1141  :( LOW ? j1 : 0UL ) );
1142  const size_t iend( ( IsUpper_v<MT4> )
1143  ?( ( IsStrictlyUpper_v<MT4> )
1144  ?( UPP ? min(j1+1UL,j) : j )
1145  :( UPP ? min(j1,j)+1UL : j+1UL ) )
1146  :( UPP ? j1+1UL : A.rows() ) );
1147 
1148  if( ( LOW || UPP ) && ( ibegin >= iend ) ) continue;
1149  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1150 
1151  const size_t inum( iend - ibegin );
1152  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1153  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1154 
1155  size_t i( ibegin );
1156 
1157  for( ; i<ipos; i+=4UL ) {
1158  C(i ,j1) += A(i ,j) * v1;
1159  C(i+1UL,j1) += A(i+1UL,j) * v1;
1160  C(i+2UL,j1) += A(i+2UL,j) * v1;
1161  C(i+3UL,j1) += A(i+3UL,j) * v1;
1162  }
1163  for( ; i<iend; ++i ) {
1164  C(i,j1) += A(i,j) * v1;
1165  }
1166  }
1167  }
1168  }
1170  //**********************************************************************************************
1171 
1172  //**Default addition assignment to dense matrices (large matrices)******************************
1186  template< typename MT3 // Type of the left-hand side target matrix
1187  , typename MT4 // Type of the left-hand side matrix operand
1188  , typename MT5 > // Type of the right-hand side matrix operand
1189  static inline auto selectLargeAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1190  -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
1191  {
1192  selectDefaultAddAssignKernel( C, A, B );
1193  }
1195  //**********************************************************************************************
1196 
1197  //**Optimized addition assignment to dense matrices (large matrices)****************************
1212  template< typename MT3 // Type of the left-hand side target matrix
1213  , typename MT4 // Type of the left-hand side matrix operand
1214  , typename MT5 > // Type of the right-hand side matrix operand
1215  static inline auto selectLargeAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1216  -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1217  {
1219 
1220  const ForwardFunctor fwd;
1221 
1222  const OppositeType_t<MT5> tmp( serial( B ) );
1223  addAssign( C, fwd( A * tmp ) );
1224  }
1226  //**********************************************************************************************
1227 
1228  //**Restructuring addition assignment***********************************************************
1243  template< typename MT // Type of the target matrix
1244  , bool SO > // Storage order of the target matrix
1245  friend inline auto addAssign( Matrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1246  -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1247  {
1249 
1250  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1251  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1252 
1253  const ForwardFunctor fwd;
1254 
1255  addAssign( ~lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1256  }
1258  //**********************************************************************************************
1259 
1260  //**Addition assignment to sparse matrices******************************************************
1261  // No special implementation for the addition assignment to sparse matrices.
1262  //**********************************************************************************************
1263 
1264  //**Subtraction assignment to dense matrices****************************************************
1277  template< typename MT // Type of the target dense matrix
1278  , bool SO > // Storage order of the target dense matrix
1279  friend inline auto subAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1280  -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1281  {
1283 
1284  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1285  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1286 
1287  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1288  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1289 
1290  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1291  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1292  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1293  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1294  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1295  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1296 
1297  TDMatSMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1298  }
1300  //**********************************************************************************************
1301 
1302  //**Subtraction assignment to dense matrices (kernel selection)*********************************
1313  template< typename MT3 // Type of the left-hand side target matrix
1314  , typename MT4 // Type of the left-hand side matrix operand
1315  , typename MT5 > // Type of the right-hand side matrix operand
1316  static inline void selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1317  {
1318  if( C.rows() * C.columns() < TDMATSMATMULT_THRESHOLD )
1319  selectSmallSubAssignKernel( C, A, B );
1320  else
1321  selectLargeSubAssignKernel( C, A, B );
1322  }
1324  //**********************************************************************************************
1325 
1326  //**Default subtraction assignment to dense matrices********************************************
1341  template< typename MT3 // Type of the left-hand side target matrix
1342  , typename MT4 // Type of the left-hand side matrix operand
1343  , typename MT5 > // Type of the right-hand side matrix operand
1344  static inline void selectDefaultSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1345  {
1346  size_t i( 0UL );
1347 
1348  for( size_t j=0UL; j<B.rows(); ++j )
1349  {
1350  auto element( B.begin(j) );
1351  const auto end( B.end(j) );
1352 
1353  if( IsDiagonal_v<MT4> )
1354  {
1355  for( ; element!=end; ++element ) {
1356  C(j,element->index()) -= A(j,j) * element->value();
1357  }
1358  }
1359  else
1360  {
1361  for( ; element!=end; ++element )
1362  {
1363  const size_t j1( element->index() );
1364 
1365  const size_t ibegin( ( IsLower_v<MT4> )
1366  ?( ( IsStrictlyLower_v<MT4> )
1367  ?( LOW ? max(j1,j+1UL) : j+1UL )
1368  :( LOW ? max(j1,j) : j ) )
1369  :( LOW ? j1 : 0UL ) );
1370  const size_t iend( ( IsUpper_v<MT4> )
1371  ?( ( IsStrictlyUpper_v<MT4> )
1372  ?( UPP ? min(j1+1UL,j) : j )
1373  :( UPP ? min(j1,j)+1UL : j+1UL ) )
1374  :( UPP ? j1+1UL : A.rows() ) );
1375 
1376  if( ( LOW || UPP ) && ( ibegin >= iend ) ) continue;
1377  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1378 
1379  const size_t inum( iend - ibegin );
1380  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1381  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1382 
1383  for( i=ibegin; i<ipos; i+=4UL ) {
1384  C(i ,j1) -= A(i ,j) * element->value();
1385  C(i+1UL,j1) -= A(i+1UL,j) * element->value();
1386  C(i+2UL,j1) -= A(i+2UL,j) * element->value();
1387  C(i+3UL,j1) -= A(i+3UL,j) * element->value();
1388  }
1389  for( ; i<iend; ++i ) {
1390  C(i,j1) -= A(i,j) * element->value();
1391  }
1392  }
1393  }
1394  }
1395  }
1397  //**********************************************************************************************
1398 
1399  //**Default subtraction assignment to dense matrices (small matrices)***************************
1413  template< typename MT3 // Type of the left-hand side target matrix
1414  , typename MT4 // Type of the left-hand side matrix operand
1415  , typename MT5 > // Type of the right-hand side matrix operand
1416  static inline auto selectSmallSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1417  -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
1418  {
1419  selectDefaultSubAssignKernel( C, A, B );
1420  }
1422  //**********************************************************************************************
1423 
1424  //**Optimized subtraction assignment to dense matrices (small matrices)*************************
1439  template< typename MT3 // Type of the left-hand side target matrix
1440  , typename MT4 // Type of the left-hand side matrix operand
1441  , typename MT5 > // Type of the right-hand side matrix operand
1442  static inline auto selectSmallSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1443  -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1444  {
1445  for( size_t j=0UL; j<B.rows(); ++j )
1446  {
1447  auto element( B.begin(j) );
1448  const auto end( B.end(j) );
1449 
1450  const size_t nonzeros( B.nonZeros(j) );
1451  const size_t kpos( nonzeros & size_t(-4) );
1452  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1453 
1454  for( size_t k=0UL; k<kpos; k+=4UL )
1455  {
1456  const size_t j1( element->index() );
1457  const ET2 v1( element->value() );
1458  ++element;
1459  const size_t j2( element->index() );
1460  const ET2 v2( element->value() );
1461  ++element;
1462  const size_t j3( element->index() );
1463  const ET2 v3( element->value() );
1464  ++element;
1465  const size_t j4( element->index() );
1466  const ET2 v4( element->value() );
1467  ++element;
1468 
1469  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1470 
1471  const size_t ibegin( ( IsLower_v<MT4> )
1472  ?( ( IsStrictlyLower_v<MT4> )
1473  ?( LOW ? max(j1,j+1UL) : j+1UL )
1474  :( LOW ? max(j1,j) : j ) )
1475  :( LOW ? j1 : 0UL ) );
1476  const size_t iend( ( IsUpper_v<MT4> )
1477  ?( ( IsStrictlyUpper_v<MT4> )
1478  ?( UPP ? min(j4+1UL,j) : j )
1479  :( UPP ? min(j4,j)+1UL : j+1UL ) )
1480  :( UPP ? j4+1UL : A.rows() ) );
1481 
1482  if( ( LOW || UPP ) && ( ibegin >= iend ) ) continue;
1483  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1484 
1485  const size_t inum( iend - ibegin );
1486  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1487  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1488 
1489  size_t i( ibegin );
1490 
1491  for( ; i<ipos; i+=4UL ) {
1492  C(i ,j1) -= A(i ,j) * v1;
1493  C(i+1UL,j1) -= A(i+1UL,j) * v1;
1494  C(i+2UL,j1) -= A(i+2UL,j) * v1;
1495  C(i+3UL,j1) -= A(i+3UL,j) * v1;
1496  C(i ,j2) -= A(i ,j) * v2;
1497  C(i+1UL,j2) -= A(i+1UL,j) * v2;
1498  C(i+2UL,j2) -= A(i+2UL,j) * v2;
1499  C(i+3UL,j2) -= A(i+3UL,j) * v2;
1500  C(i ,j3) -= A(i ,j) * v3;
1501  C(i+1UL,j3) -= A(i+1UL,j) * v3;
1502  C(i+2UL,j3) -= A(i+2UL,j) * v3;
1503  C(i+3UL,j3) -= A(i+3UL,j) * v3;
1504  C(i ,j4) -= A(i ,j) * v4;
1505  C(i+1UL,j4) -= A(i+1UL,j) * v4;
1506  C(i+2UL,j4) -= A(i+2UL,j) * v4;
1507  C(i+3UL,j4) -= A(i+3UL,j) * v4;
1508  }
1509  for( ; i<iend; ++i ) {
1510  C(i,j1) -= A(i,j) * v1;
1511  C(i,j2) -= A(i,j) * v2;
1512  C(i,j3) -= A(i,j) * v3;
1513  C(i,j4) -= A(i,j) * v4;
1514  }
1515  }
1516 
1517  for( ; element!=end; ++element )
1518  {
1519  const size_t j1( element->index() );
1520  const ET2 v1( element->value() );
1521 
1522  const size_t ibegin( ( IsLower_v<MT4> )
1523  ?( ( IsStrictlyLower_v<MT4> )
1524  ?( LOW ? max(j1,j+1UL) : j+1UL )
1525  :( LOW ? max(j1,j) : j ) )
1526  :( LOW ? j1 : 0UL ) );
1527  const size_t iend( ( IsUpper_v<MT4> )
1528  ?( ( IsStrictlyUpper_v<MT4> )
1529  ?( UPP ? min(j1+1UL,j) : j )
1530  :( UPP ? min(j1,j)+1UL : j+1UL ) )
1531  :( UPP ? j1+1UL : A.rows() ) );
1532 
1533  if( ( LOW || UPP ) && ( ibegin >= iend ) ) continue;
1534  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1535 
1536  const size_t inum( iend - ibegin );
1537  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1538  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1539 
1540  size_t i( ibegin );
1541 
1542  for( ; i<ipos; i+=4UL ) {
1543  C(i ,j1) -= A(i ,j) * v1;
1544  C(i+1UL,j1) -= A(i+1UL,j) * v1;
1545  C(i+2UL,j1) -= A(i+2UL,j) * v1;
1546  C(i+3UL,j1) -= A(i+3UL,j) * v1;
1547  }
1548  for( ; i<iend; ++i ) {
1549  C(i,j1) -= A(i,j) * v1;
1550  }
1551  }
1552  }
1553  }
1555  //**********************************************************************************************
1556 
1557  //**Default subtraction assignment to dense matrices (large matrices)***************************
1571  template< typename MT3 // Type of the left-hand side target matrix
1572  , typename MT4 // Type of the left-hand side matrix operand
1573  , typename MT5 > // Type of the right-hand side matrix operand
1574  static inline auto selectLargeSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1575  -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
1576  {
1577  selectDefaultSubAssignKernel( C, A, B );
1578  }
1580  //**********************************************************************************************
1581 
1582  //**Optimized subtraction assignment to dense matrices (large matrices)*************************
1597  template< typename MT3 // Type of the left-hand side target matrix
1598  , typename MT4 // Type of the left-hand side matrix operand
1599  , typename MT5 > // Type of the right-hand side matrix operand
1600  static inline auto selectLargeSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1601  -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1602  {
1604 
1605  const ForwardFunctor fwd;
1606 
1607  const OppositeType_t<MT5> tmp( serial( B ) );
1608  subAssign( C, fwd( A * tmp ) );
1609  }
1611  //**********************************************************************************************
1612 
1613  //**Restructuring subtraction assignment********************************************************
1628  template< typename MT // Type of the target matrix
1629  , bool SO > // Storage order of the target matrix
1630  friend inline auto subAssign( Matrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1631  -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1632  {
1634 
1636 
1637  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1638  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1639 
1640  const ForwardFunctor fwd;
1641 
1642  subAssign( ~lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1643  }
1645  //**********************************************************************************************
1646 
1647  //**Subtraction assignment to sparse matrices***************************************************
1648  // No special implementation for the subtraction assignment to sparse matrices.
1649  //**********************************************************************************************
1650 
1651  //**Schur product assignment to dense matrices**************************************************
1664  template< typename MT // Type of the target dense matrix
1665  , bool SO > // Storage order of the target dense matrix
1666  friend inline void schurAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1667  {
1669 
1673 
1674  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1675  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1676 
1677  const ResultType tmp( serial( rhs ) );
1678  schurAssign( ~lhs, tmp );
1679  }
1681  //**********************************************************************************************
1682 
1683  //**Schur product assignment to sparse matrices*************************************************
1684  // No special implementation for the Schur product assignment to sparse matrices.
1685  //**********************************************************************************************
1686 
1687  //**Multiplication assignment to dense matrices*************************************************
1688  // No special implementation for the multiplication assignment to dense matrices.
1689  //**********************************************************************************************
1690 
1691  //**Multiplication assignment to sparse matrices************************************************
1692  // No special implementation for the multiplication assignment to sparse matrices.
1693  //**********************************************************************************************
1694 
1695  //**SMP assignment to dense matrices************************************************************
1711  template< typename MT // Type of the target dense matrix
1712  , bool SO > // Storage order of the target dense matrix
1713  friend inline auto smpAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1714  -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1715  {
1717 
1718  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1719  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1720 
1721  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1722  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1723 
1724  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1725  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1726  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1727  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1728  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1729  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1730 
1731  smpAssign( ~lhs, A * B );
1732  }
1734  //**********************************************************************************************
1735 
1736  //**SMP assignment to sparse matrices***********************************************************
1752  template< typename MT // Type of the target sparse matrix
1753  , bool SO > // Storage order of the target sparse matrix
1754  friend inline auto smpAssign( SparseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1755  -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1756  {
1758 
1759  using TmpType = If_t< SO, ResultType, OppositeType >;
1760 
1767 
1768  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1769  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1770 
1771  const ForwardFunctor fwd;
1772 
1773  const TmpType tmp( rhs );
1774  smpAssign( ~lhs, fwd( tmp ) );
1775  }
1777  //**********************************************************************************************
1778 
1779  //**Restructuring SMP assignment****************************************************************
1794  template< typename MT // Type of the target matrix
1795  , bool SO > // Storage order of the target matrix
1796  friend inline auto smpAssign( Matrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1797  -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1798  {
1800 
1801  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1802  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1803 
1804  const ForwardFunctor fwd;
1805 
1806  smpAssign( ~lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1807  }
1809  //**********************************************************************************************
1810 
1811  //**SMP addition assignment to dense matrices***************************************************
1827  template< typename MT // Type of the target dense matrix
1828  , bool SO > // Storage order of the target sparse matrix
1829  friend inline auto smpAddAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1830  -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1831  {
1833 
1834  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1835  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1836 
1837  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1838  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1839 
1840  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1841  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1842  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1843  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1844  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1845  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1846 
1847  smpAddAssign( ~lhs, A * B );
1848  }
1850  //**********************************************************************************************
1851 
1852  //**Restructuring SMP addition assignment*******************************************************
1867  template< typename MT // Type of the target matrix
1868  , bool SO > // Storage order of the target matrix
1869  friend inline auto smpAddAssign( Matrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1870  -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1871  {
1873 
1875 
1876  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1877  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1878 
1879  const ForwardFunctor fwd;
1880 
1881  smpAddAssign( ~lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1882  }
1884  //**********************************************************************************************
1885 
1886  //**SMP addition assignment to sparse matrices**************************************************
1887  // No special implementation for the SMP addition assignment to sparse matrices.
1888  //**********************************************************************************************
1889 
1890  //**SMP subtraction assignment to dense matrices************************************************
1906  template< typename MT // Type of the target dense matrix
1907  , bool SO > // Storage order of the target sparse matrix
1908  friend inline auto smpSubAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1909  -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1910  {
1912 
1913  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1914  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1915 
1916  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1917  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1918 
1919  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1920  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1921  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1922  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1923  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1924  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1925 
1926  smpSubAssign( ~lhs, A * B );
1927  }
1929  //**********************************************************************************************
1930 
1931  //**Restructuring SMP subtraction assignment****************************************************
1946  template< typename MT // Type of the target matrix
1947  , bool SO > // Storage order of the target matrix
1948  friend inline auto smpSubAssign( Matrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1949  -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1950  {
1952 
1954 
1955  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1956  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1957 
1958  const ForwardFunctor fwd;
1959 
1960  smpSubAssign( ~lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1961  }
1963  //**********************************************************************************************
1964 
1965  //**SMP subtraction assignment to sparse matrices***********************************************
1966  // No special implementation for the SMP subtraction assignment to sparse matrices.
1967  //**********************************************************************************************
1968 
1969  //**SMP Schur product assignment to dense matrices**********************************************
1982  template< typename MT // Type of the target dense matrix
1983  , bool SO > // Storage order of the target sparse matrix
1984  friend inline void smpSchurAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1985  {
1987 
1991 
1992  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1993  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1994 
1995  const ResultType tmp( rhs );
1996  smpSchurAssign( ~lhs, tmp );
1997  }
1999  //**********************************************************************************************
2000 
2001  //**SMP Schur product assignment to sparse matrices*********************************************
2002  // No special implementation for the SMP Schur product assignment to sparse matrices.
2003  //**********************************************************************************************
2004 
2005  //**SMP multiplication assignment to dense matrices*********************************************
2006  // No special implementation for the SMP multiplication assignment to dense matrices.
2007  //**********************************************************************************************
2008 
2009  //**SMP multiplication assignment to sparse matrices********************************************
2010  // No special implementation for the SMP multiplication assignment to sparse matrices.
2011  //**********************************************************************************************
2012 
2013  //**Compile time checks*************************************************************************
2022  //**********************************************************************************************
2023 };
2024 //*************************************************************************************************
2025 
2026 
2027 
2028 
2029 //=================================================================================================
2030 //
2031 // GLOBAL BINARY ARITHMETIC OPERATORS
2032 //
2033 //=================================================================================================
2034 
2035 //*************************************************************************************************
2048 template< typename MT1 // Type of the left-hand side dense matrix
2049  , typename MT2 // Type of the right-hand side sparse matrix
2050  , DisableIf_t< ( IsIdentity_v<MT2> &&
2051  IsSame_v< ElementType_t<MT1>, ElementType_t<MT2> > ) ||
2052  IsZero_v<MT2> >* = nullptr >
2053 inline const TDMatSMatMultExpr<MT1,MT2,false,false,false,false>
2054  tdmatsmatmult( const DenseMatrix<MT1,true>& lhs, const SparseMatrix<MT2,false>& rhs )
2055 {
2057 
2058  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == (~rhs).rows(), "Invalid matrix sizes" );
2059 
2060  return TDMatSMatMultExpr<MT1,MT2,false,false,false,false>( ~lhs, ~rhs );
2061 }
2063 //*************************************************************************************************
2064 
2065 
2066 //*************************************************************************************************
2080 template< typename MT1 // Type of the left-hand side dense matrix
2081  , typename MT2 // Type of the right-hand side sparse matrix
2082  , EnableIf_t< IsIdentity_v<MT2> &&
2083  IsSame_v< ElementType_t<MT1>, ElementType_t<MT2> > >* = nullptr >
2084 inline const MT1&
2085  tdmatsmatmult( const DenseMatrix<MT1,true>& lhs, const SparseMatrix<MT2,false>& rhs )
2086 {
2088 
2089  UNUSED_PARAMETER( rhs );
2090 
2091  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == (~rhs).rows(), "Invalid matrix sizes" );
2092 
2093  return (~lhs);
2094 }
2096 //*************************************************************************************************
2097 
2098 
2099 //*************************************************************************************************
2112 template< typename MT1 // Type of the left-hand side dense matrix
2113  , typename MT2 // Type of the right-hand side sparse matrix
2114  , EnableIf_t< IsZero_v<MT2> >* = nullptr >
2115 inline decltype(auto)
2116  tdmatsmatmult( const DenseMatrix<MT1,true>& lhs, const SparseMatrix<MT2,false>& rhs )
2117 {
2119 
2120  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == (~rhs).rows(), "Invalid matrix sizes" );
2121 
2122  using ReturnType = const MultTrait_t< ResultType_t<MT1>, ResultType_t<MT2> >;
2123 
2126 
2127  return ReturnType( (~lhs).rows(), (~rhs).columns() );
2128 }
2130 //*************************************************************************************************
2131 
2132 
2133 //*************************************************************************************************
2163 template< typename MT1 // Type of the left-hand side dense matrix
2164  , typename MT2 > // Type of the right-hand side sparse matrix
2165 inline decltype(auto)
2166  operator*( const DenseMatrix<MT1,true>& lhs, const SparseMatrix<MT2,false>& rhs )
2167 {
2169 
2170  if( (~lhs).columns() != (~rhs).rows() ) {
2171  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
2172  }
2173 
2174  return tdmatsmatmult( ~lhs, ~rhs );
2175 }
2176 //*************************************************************************************************
2177 
2178 
2179 
2180 
2181 //=================================================================================================
2182 //
2183 // GLOBAL FUNCTIONS
2184 //
2185 //=================================================================================================
2186 
2187 //*************************************************************************************************
2212 template< typename MT1 // Type of the left-hand side dense matrix
2213  , typename MT2 // Type of the right-hand side dense matrix
2214  , bool SF // Symmetry flag
2215  , bool HF // Hermitian flag
2216  , bool LF // Lower flag
2217  , bool UF > // Upper flag
2218 inline decltype(auto) declsym( const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2219 {
2221 
2222  if( !isSquare( dm ) ) {
2223  BLAZE_THROW_INVALID_ARGUMENT( "Invalid symmetric matrix specification" );
2224  }
2225 
2226  using ReturnType = const TDMatSMatMultExpr<MT1,MT2,true,HF,LF,UF>;
2227  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2228 }
2230 //*************************************************************************************************
2231 
2232 
2233 //*************************************************************************************************
2258 template< typename MT1 // Type of the left-hand side dense matrix
2259  , typename MT2 // Type of the right-hand side dense matrix
2260  , bool SF // Symmetry flag
2261  , bool HF // Hermitian flag
2262  , bool LF // Lower flag
2263  , bool UF > // Upper flag
2264 inline decltype(auto) declherm( const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2265 {
2267 
2268  if( !isSquare( dm ) ) {
2269  BLAZE_THROW_INVALID_ARGUMENT( "Invalid Hermitian matrix specification" );
2270  }
2271 
2272  using ReturnType = const TDMatSMatMultExpr<MT1,MT2,SF,true,LF,UF>;
2273  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2274 }
2276 //*************************************************************************************************
2277 
2278 
2279 //*************************************************************************************************
2304 template< typename MT1 // Type of the left-hand side dense matrix
2305  , typename MT2 // Type of the right-hand side dense matrix
2306  , bool SF // Symmetry flag
2307  , bool HF // Hermitian flag
2308  , bool LF // Lower flag
2309  , bool UF > // Upper flag
2310 inline decltype(auto) decllow( const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2311 {
2313 
2314  if( !isSquare( dm ) ) {
2315  BLAZE_THROW_INVALID_ARGUMENT( "Invalid lower matrix specification" );
2316  }
2317 
2318  using ReturnType = const TDMatSMatMultExpr<MT1,MT2,SF,HF,true,UF>;
2319  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2320 }
2322 //*************************************************************************************************
2323 
2324 
2325 //*************************************************************************************************
2350 template< typename MT1 // Type of the left-hand side dense matrix
2351  , typename MT2 // Type of the right-hand side dense matrix
2352  , bool SF // Symmetry flag
2353  , bool HF // Hermitian flag
2354  , bool LF // Lower flag
2355  , bool UF > // Upper flag
2356 inline decltype(auto) declupp( const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2357 {
2359 
2360  if( !isSquare( dm ) ) {
2361  BLAZE_THROW_INVALID_ARGUMENT( "Invalid upper matrix specification" );
2362  }
2363 
2364  using ReturnType = const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,true>;
2365  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2366 }
2368 //*************************************************************************************************
2369 
2370 
2371 //*************************************************************************************************
2396 template< typename MT1 // Type of the left-hand side dense matrix
2397  , typename MT2 // Type of the right-hand side dense matrix
2398  , bool SF // Symmetry flag
2399  , bool HF // Hermitian flag
2400  , bool LF // Lower flag
2401  , bool UF > // Upper flag
2402 inline decltype(auto) decldiag( const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2403 {
2405 
2406  if( !isSquare( dm ) ) {
2407  BLAZE_THROW_INVALID_ARGUMENT( "Invalid diagonal matrix specification" );
2408  }
2409 
2410  using ReturnType = const TDMatSMatMultExpr<MT1,MT2,SF,HF,true,true>;
2411  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2412 }
2414 //*************************************************************************************************
2415 
2416 
2417 
2418 
2419 //=================================================================================================
2420 //
2421 // SIZE SPECIALIZATIONS
2422 //
2423 //=================================================================================================
2424 
2425 //*************************************************************************************************
2427 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2428 struct Size< TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 0UL >
2429  : public Size<MT1,0UL>
2430 {};
2431 
2432 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2433 struct Size< TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 1UL >
2434  : public Size<MT2,1UL>
2435 {};
2437 //*************************************************************************************************
2438 
2439 
2440 
2441 
2442 //=================================================================================================
2443 //
2444 // ISALIGNED SPECIALIZATIONS
2445 //
2446 //=================================================================================================
2447 
2448 //*************************************************************************************************
2450 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2451 struct IsAligned< TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2452  : public IsAligned<MT1>
2453 {};
2455 //*************************************************************************************************
2456 
2457 } // namespace blaze
2458 
2459 #endif
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:329
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Header file for auxiliary alias declarations.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:133
Headerfile for the generic min algorithm.
Header file for the blaze::checked and blaze::unchecked instances.
Header file for the decldiag trait.
decltype(auto) decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as diagonal.
Definition: DMatDeclDiagExpr.h:975
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: TDMatSMatMultExpr.h:360
ElementType_t< ResultType > ElementType
Resulting element type.
Definition: TDMatSMatMultExpr.h:248
Header file for the UNUSED_PARAMETER function template.
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: TDMatSMatMultExpr.h:295
Header file for basic type definitions.
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: TDMatSMatMultExpr.h:344
LeftOperand leftOperand() const noexcept
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatSMatMultExpr.h:380
typename If< Condition, T1, T2 >::Type If_t
Auxiliary alias declaration for the If class template.The If_t alias declaration provides a convenien...
Definition: If.h:109
Header file for the declherm trait.
CompositeType_t< MT2 > CT2
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:138
static constexpr bool simdEnabled
Compilation switch for the expression template evaluation strategy.
Definition: TDMatSMatMultExpr.h:267
typename T::ResultType ResultType_t
Alias declaration for nested ResultType type definitions.The ResultType_t alias declaration provides ...
Definition: Aliases.h:390
Header file for the serial shim.
Header file for the IsDiagonal type trait.
Base template for the DeclUppTrait class.
Definition: DeclUppTrait.h:134
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the DeclUpp functor.
ElementType_t< RT2 > ET2
Element type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:136
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:372
Header file for the IsColumnMajorMatrix type trait.
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:591
static constexpr bool smpAssignable
Compilation flag for SMP assignments.
Definition: CompressedMatrix.h:3113
RightOperand rightOperand() const noexcept
Returns the right-hand side sparse matrix operand.
Definition: TDMatSMatMultExpr.h:390
constexpr Unchecked unchecked
Global Unchecked instance.The blaze::unchecked instance is an optional token for the creation of view...
Definition: Check.h:138
Constraint on the data type.
Header file for the IsIdentity type trait.
decltype(auto) declupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as upper.
Definition: DMatDeclUppExpr.h:1002
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Header file for the reset shim.
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Expression object for transpose dense matrix-sparse matrix multiplications.The TDMatSMatMultExpr clas...
Definition: Forward.h:160
constexpr void UNUSED_PARAMETER(const Args &...)
Suppression of unused parameter warnings.
Definition: Unused.h:81
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatSMatMultExpr.h:250
constexpr size_t columns(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of columns of the matrix.
Definition: Matrix.h:514
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:80
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:137
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.The ElementType_t alias declaration provide...
Definition: Aliases.h:170
If_t< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: TDMatSMatMultExpr.h:262
static constexpr bool UPP
Flag for upper matrices.
Definition: TDMatSMatMultExpr.h:155
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: TDMatSMatMultExpr.h:442
Constraint on the data type.
Constraint on the data type.
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TDMatSMatMultExpr.h:402
TransposeType_t< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TDMatSMatMultExpr.h:247
Headerfile for the generic max algorithm.
Header file for the DisableIf class template.
If_t< IsExpression_v< MT1 >, const MT1, const MT1 &> LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:253
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Header file for the IsSymmetric type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:58
Header file for the DeclLow functor.
Header file for the If class template.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_ZERO_TYPE(T)
Constraint on the data type.In case the given data type T is not a zero vector or matrix type...
Definition: Zero.h:61
Generic wrapper for the decllow() function.
Definition: DeclLow.h:58
decltype(auto) min(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise minimum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1147
Header file for the decllow trait.
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the DenseMatrix base class.
decltype(auto) decllow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as lower.
Definition: DMatDeclLowExpr.h:1002
Header file for the IsLower type trait.
Header file for the IsAligned type trait.
Generic wrapper for the null function.
Definition: Noop.h:59
Header file for the IsTriangular type trait.
Base template for the DeclSymTrait class.
Definition: DeclSymTrait.h:134
Constraints on the storage order of matrix types.
static constexpr bool evaluateLeft
Compilation switch for the composite type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:143
Header file for the exception macros of the math module.
decltype(auto) max(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise maximum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1179
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:438
OppositeType_t< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: TDMatSMatMultExpr.h:246
Header file for the DeclDiag functor.
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatSMatMultExpr.h:424
Constraint on the data type.
Header file for all forward declarations for expression class templates.
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:103
If_t< IsExpression_v< MT2 >, const MT2, const MT2 &> RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:256
typename T::OppositeType OppositeType_t
Alias declaration for nested OppositeType type definitions.The OppositeType_t alias declaration provi...
Definition: Aliases.h:270
Header file for the conjugate shim.
ResultType_t< MT2 > RT2
Result type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:134
Header file for the declupp trait.
ElementType_t< RT1 > ET1
Element type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:135
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type, a compilation error is created.
Definition: Symmetric.h:79
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatSMatMultExpr.h:249
typename T::TransposeType TransposeType_t
Alias declaration for nested TransposeType type definitions.The TransposeType_t alias declaration pro...
Definition: Aliases.h:470
Header file for run time assertion macros.
TDMatSMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the TDMatSMatMultExpr class.
Definition: TDMatSMatMultExpr.h:280
Base template for the DeclHermTrait class.
Definition: DeclHermTrait.h:134
ResultType_t< MT1 > RT1
Result type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:133
typename T::CompositeType CompositeType_t
Alias declaration for nested CompositeType type definitions.The CompositeType_t alias declaration pro...
Definition: Aliases.h:90
Base template for the MultTrait class.
Definition: MultTrait.h:146
auto smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:131
If_t< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatSMatMultExpr.h:259
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:133
Header file for the IsZero type trait.
Header file for the declsym trait.
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:94
static constexpr bool evaluateRight
Compilation switch for the composite type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:148
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatSMatMultExpr.h:441
decltype(auto) declsym(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as symmetric.
Definition: DMatDeclSymExpr.h:1002
Header file for the isDefault shim.
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
Constraint on the data type.
Constraints on the storage order of matrix types.
Generic wrapper for the declherm() function.
Definition: DeclHerm.h:58
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:808
Header file for the Noop functor.
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: TDMatSMatMultExpr.h:370
auto smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:194
constexpr size_t rows(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of rows of the matrix.
Definition: Matrix.h:498
CompositeType_t< MT1 > CT1
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:137
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:84
typename If_t< HERM, DeclHermTrait< MultTrait_t< RT1, RT2 > >, If_t< SYM, DeclSymTrait< MultTrait_t< RT1, RT2 > >, If_t< LOW, If_t< UPP, DeclDiagTrait< MultTrait_t< RT1, RT2 > >, DeclLowTrait< MultTrait_t< RT1, RT2 > > >, If_t< UPP, DeclUppTrait< MultTrait_t< RT1, RT2 > >, MultTrait< RT1, RT2 > > > > >::Type ResultType
Result type for expression template evaluations.
Definition: TDMatSMatMultExpr.h:244
Generic wrapper for the declupp() function.
Definition: DeclUpp.h:58
const Type & ReturnType
Return type for expression template evaluations.
Definition: CompressedMatrix.h:3081
decltype(auto) trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:765
static constexpr bool smpAssignable
Compilation switch for the expression template assignment strategy.
Definition: TDMatSMatMultExpr.h:270
Base template for the DeclLowTrait class.
Definition: DeclLowTrait.h:134
decltype(auto) declherm(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as Hermitian.
Definition: DMatDeclHermExpr.h:1002
bool isAliased(const T *alias) const
Returns whether the expression is aliased with the given address alias.
Definition: TDMatSMatMultExpr.h:414
Header file for the IsComputation type trait class.
Header file for the IsBuiltin type trait.
auto smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:162
static constexpr bool SYM
Flag for symmetric matrices.
Definition: TDMatSMatMultExpr.h:152
Header file for the IntegralConstant class template.
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TDMatSMatMultExpr.h:434
Generic wrapper for the decldiag() function.
Definition: DeclDiag.h:58
Header file for the DeclHerm functor.
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:631
static constexpr bool HERM
Flag for Hermitian matrices.
Definition: TDMatSMatMultExpr.h:153
Header file for the IsUpper type trait.
typename DisableIf< Condition, T >::Type DisableIf_t
Auxiliary type for the DisableIf class template.The DisableIf_t alias declaration provides a convenie...
Definition: DisableIf.h:138
decltype(auto) conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition: DMatMapExpr.h:1326
Constraint on the data type.
Generic wrapper for the declsym() function.
Definition: DeclSym.h:58
Base template for the DeclDiagTrait class.
Definition: DeclDiagTrait.h:134
bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition: Matrix.h:951
Header file for the IsResizable type trait.
Header file for the Size type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_ZERO_TYPE(T)
Constraint on the data type.In case the given data type T is a zero vector or matrix type...
Definition: Zero.h:81
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the DeclSym functor.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:61
Header file for the IsExpression type trait class.
Header file for the function trace functionality.
static constexpr bool LOW
Flag for lower matrices.
Definition: TDMatSMatMultExpr.h:154