All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
TDMatSMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATSMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATSMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <stdexcept>
54 #include <blaze/math/shims/Reset.h>
85 #include <blaze/util/Assert.h>
87 #include <blaze/util/DisableIf.h>
88 #include <blaze/util/EnableIf.h>
89 #include <blaze/util/InvalidType.h>
91 #include <blaze/util/SelectType.h>
92 #include <blaze/util/Types.h>
94 
95 
96 namespace blaze {
97 
98 //=================================================================================================
99 //
100 // CLASS TDMATSMATMULTEXPR
101 //
102 //=================================================================================================
103 
104 //*************************************************************************************************
111 template< typename MT1 // Type of the left-hand side dense matrix
112  , typename MT2 > // Type of the right-hand side sparse matrix
113 class TDMatSMatMultExpr : public DenseMatrix< TDMatSMatMultExpr<MT1,MT2>, true >
114  , private MatMatMultExpr
115  , private Computation
116 {
117  private:
118  //**Type definitions****************************************************************************
119  typedef typename MT1::ResultType RT1;
120  typedef typename MT2::ResultType RT2;
121  typedef typename RT1::ElementType ET1;
122  typedef typename RT2::ElementType ET2;
123  typedef typename MT1::CompositeType CT1;
124  typedef typename MT2::CompositeType CT2;
125  //**********************************************************************************************
126 
127  //**********************************************************************************************
130  //**********************************************************************************************
131 
132  //**********************************************************************************************
134  enum { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
135  //**********************************************************************************************
136 
137  //**********************************************************************************************
139 
146  template< typename T1, typename T2, typename T3 >
147  struct CanExploitSymmetry {
148  enum { value = ( IsRowMajorMatrix<T1>::value && IsSymmetric<T2>::value ) ||
149  ( IsColumnMajorMatrix<T1>::value && IsSymmetric<T3>::value ) };
150  };
152  //**********************************************************************************************
153 
154  //**********************************************************************************************
156 
160  template< typename T1, typename T2, typename T3 >
161  struct IsEvaluationRequired {
162  enum { value = ( evaluateLeft || evaluateRight ) &&
163  !CanExploitSymmetry<T1,T2,T3>::value };
164  };
166  //**********************************************************************************************
167 
168  //**********************************************************************************************
170 
174  template< typename T1, typename T2, typename T3 >
175  struct UseOptimizedKernel {
176  enum { value = !IsResizable<typename T1::ElementType>::value };
177  };
179  //**********************************************************************************************
180 
181  //**********************************************************************************************
183 
186  template< typename T1, typename T2, typename T3 >
187  struct UseDefaultKernel {
188  enum { value = !UseOptimizedKernel<T1,T2,T3>::value };
189  };
191  //**********************************************************************************************
192 
193  public:
194  //**Type definitions****************************************************************************
200  typedef const ElementType ReturnType;
201  typedef const ResultType CompositeType;
202 
204  typedef typename SelectType< IsExpression<MT1>::value, const MT1, const MT1& >::Type LeftOperand;
205 
207  typedef typename SelectType< IsExpression<MT2>::value, const MT2, const MT2& >::Type RightOperand;
208 
211 
214  //**********************************************************************************************
215 
216  //**Compilation flags***************************************************************************
218  enum { vectorizable = 0 };
219 
221  enum { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
222  !evaluateRight && MT2::smpAssignable };
223  //**********************************************************************************************
224 
225  //**Constructor*********************************************************************************
231  explicit inline TDMatSMatMultExpr( const MT1& lhs, const MT2& rhs )
232  : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
233  , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
234  {
235  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
236  }
237  //**********************************************************************************************
238 
239  //**Access operator*****************************************************************************
246  inline ReturnType operator()( size_t i, size_t j ) const {
247  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
248  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
249 
250  ElementType tmp;
251 
252  if( lhs_.columns() != 0 ) {
253  tmp = lhs_(i,0) * rhs_(0,j);
254  for( size_t k=1; k<lhs_.columns(); ++k ) {
255  tmp += lhs_(i,k) * rhs_(k,j);
256  }
257  }
258  else {
259  reset( tmp );
260  }
261 
262  return tmp;
263  }
264  //**********************************************************************************************
265 
266  //**Rows function*******************************************************************************
271  inline size_t rows() const {
272  return lhs_.rows();
273  }
274  //**********************************************************************************************
275 
276  //**Columns function****************************************************************************
281  inline size_t columns() const {
282  return rhs_.columns();
283  }
284  //**********************************************************************************************
285 
286  //**Left operand access*************************************************************************
291  inline LeftOperand leftOperand() const {
292  return lhs_;
293  }
294  //**********************************************************************************************
295 
296  //**Right operand access************************************************************************
301  inline RightOperand rightOperand() const {
302  return rhs_;
303  }
304  //**********************************************************************************************
305 
306  //**********************************************************************************************
312  template< typename T >
313  inline bool canAlias( const T* alias ) const {
314  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
315  }
316  //**********************************************************************************************
317 
318  //**********************************************************************************************
324  template< typename T >
325  inline bool isAliased( const T* alias ) const {
326  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
327  }
328  //**********************************************************************************************
329 
330  //**********************************************************************************************
335  inline bool isAligned() const {
336  return lhs_.isAligned();
337  }
338  //**********************************************************************************************
339 
340  //**********************************************************************************************
345  inline bool canSMPAssign() const {
346  return ( columns() > SMP_TDMATSMATMULT_THRESHOLD );
347  }
348  //**********************************************************************************************
349 
350  private:
351  //**Member variables****************************************************************************
354  //**********************************************************************************************
355 
356  //**Assignment to row-major dense matrices******************************************************
369  template< typename MT > // Type of the target dense matrix
370  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
372  {
374 
376 
377  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
378  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
379 
380  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
381  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
382 
383  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
384  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
385  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
386  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
387  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
388  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
389 
390  TDMatSMatMultExpr::selectRowMajorAssignKernel( ~lhs, A, B );
391  }
393  //**********************************************************************************************
394 
395  //**Default assignment to row-major dense matrices**********************************************
410  template< typename MT3 // Type of the left-hand side target matrix
411  , typename MT4 // Type of the left-hand side matrix operand
412  , typename MT5 > // Type of the right-hand side matrix operand
413  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
414  selectRowMajorAssignKernel( MT3& C, const MT4& A, const MT5& B )
415  {
416  typedef typename MT5::ConstIterator ConstIterator;
417 
418  for( size_t i=0; i<A.rows(); ++i ) {
419  for( size_t j=0; j<C.columns(); ++j ) {
420  reset( C(i,j) );
421  }
422  for( size_t j=0; j<B.rows(); ++j ) {
423  ConstIterator element( B.begin(j) );
424  const ConstIterator end( B.end(j) );
425  for( ; element!=end; ++element ) {
426  if( isDefault( C(i,element->index()) ) )
427  C(i,element->index()) = A(i,j) * element->value();
428  else
429  C(i,element->index()) += A(i,j) * element->value();
430  }
431  }
432  }
433  }
435  //**********************************************************************************************
436 
437  //**Optimized assignment to row-major dense matrices********************************************
452  template< typename MT3 // Type of the left-hand side target matrix
453  , typename MT4 // Type of the left-hand side matrix operand
454  , typename MT5 > // Type of the right-hand side matrix operand
455  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
456  selectRowMajorAssignKernel( MT3& C, const MT4& A, const MT5& B )
457  {
458  typedef typename MT5::ConstIterator ConstIterator;
459 
460  const size_t last( A.rows() & size_t(-4) );
461 
462  for( size_t i=0; i<last; i+=4 ) {
463  for( size_t j=0; j<C.columns(); ++j ) {
464  reset( C(i ,j) );
465  reset( C(i+1,j) );
466  reset( C(i+2,j) );
467  reset( C(i+3,j) );
468  }
469  for( size_t j=0; j<B.rows(); ++j ) {
470  ConstIterator element( B.begin(j) );
471  const ConstIterator end( B.end(j) );
472  for( ; element!=end; ++element ) {
473  C(i ,element->index()) += A(i ,j) * element->value();
474  C(i+1,element->index()) += A(i+1,j) * element->value();
475  C(i+2,element->index()) += A(i+2,j) * element->value();
476  C(i+3,element->index()) += A(i+3,j) * element->value();
477  }
478  }
479  }
480 
481  for( size_t i=last; i<A.rows(); ++i ) {
482  for( size_t j=0; j<C.columns(); ++j ) {
483  reset( C(i,j) );
484  }
485  for( size_t j=0; j<B.rows(); ++j ) {
486  ConstIterator element( B.begin(j) );
487  const ConstIterator end( B.end(j) );
488  for( ; element!=end; ++element ) {
489  C(i,element->index()) += A(i,j) * element->value();
490  }
491  }
492  }
493  }
495  //**********************************************************************************************
496 
497  //**Assignment to column-major dense matrices***************************************************
510  template< typename MT > // Type of the target dense matrix
511  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
512  assign( DenseMatrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
513  {
515 
516  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
517  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
518 
519  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
520  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
521 
522  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
523  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
524  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
525  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
526  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
527  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
528 
529  TDMatSMatMultExpr::selectColumnMajorAssignKernel( ~lhs, A, B );
530  }
532  //**********************************************************************************************
533 
534  //**Default assignment to column-major dense matrices*******************************************
549  template< typename MT3 // Type of the left-hand side target matrix
550  , typename MT4 // Type of the left-hand side matrix operand
551  , typename MT5 > // Type of the right-hand side matrix operand
552  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
553  selectColumnMajorAssignKernel( MT3& C, const MT4& A, const MT5& B )
554  {
555  typedef typename MT5::ConstIterator ConstIterator;
556 
557  reset( C );
558 
559  for( size_t i=0; i<B.rows(); ++i ) {
560  ConstIterator element( B.begin(i) );
561  const ConstIterator end( B.end(i) );
562  for( ; element!=end; ++element ) {
563  for( size_t j=0; j<A.rows(); ++j ) {
564  if( isDefault( C(j,element->index()) ) )
565  C(j,element->index()) = A(j,i) * element->value();
566  else
567  C(j,element->index()) += A(j,i) * element->value();
568  }
569  }
570  }
571  }
573  //**********************************************************************************************
574 
575  //**Optimized assignment to column-major dense matrices*****************************************
590  template< typename MT3 // Type of the left-hand side target matrix
591  , typename MT4 // Type of the left-hand side matrix operand
592  , typename MT5 > // Type of the right-hand side matrix operand
593  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
594  selectColumnMajorAssignKernel( MT3& C, const MT4& A, const MT5& B )
595  {
596  typedef typename MT5::ConstIterator ConstIterator;
597 
598  reset( C );
599 
600  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == ( A.rows() & size_t(-4) ), "Invalid end calculation" );
601  const size_t jend( A.rows() & size_t(-4) );
602  size_t j( 0UL );
603 
604  for( size_t i=0UL; i<B.rows(); ++i ) {
605  ConstIterator element( B.begin(i) );
606  const ConstIterator end( B.end(i) );
607 
608  const size_t nonzeros( B.nonZeros(i) );
609  const size_t kend( nonzeros & size_t(-4) );
610 
611  for( size_t k=0UL; k<kend; k+=4UL )
612  {
613  const size_t i1( element->index() );
614  const ET2 v1( element->value() );
615  ++element;
616  const size_t i2( element->index() );
617  const ET2 v2( element->value() );
618  ++element;
619  const size_t i3( element->index() );
620  const ET2 v3( element->value() );
621  ++element;
622  const size_t i4( element->index() );
623  const ET2 v4( element->value() );
624  ++element;
625 
626  for( j=0UL; j<jend; j+=4UL ) {
627  C(j ,i1) += A(j ,i) * v1;
628  C(j+1UL,i1) += A(j+1UL,i) * v1;
629  C(j+2UL,i1) += A(j+2UL,i) * v1;
630  C(j+3UL,i1) += A(j+3UL,i) * v1;
631  C(j ,i2) += A(j ,i) * v2;
632  C(j+1UL,i2) += A(j+1UL,i) * v2;
633  C(j+2UL,i2) += A(j+2UL,i) * v2;
634  C(j+3UL,i2) += A(j+3UL,i) * v2;
635  C(j ,i3) += A(j ,i) * v3;
636  C(j+1UL,i3) += A(j+1UL,i) * v3;
637  C(j+2UL,i3) += A(j+2UL,i) * v3;
638  C(j+3UL,i3) += A(j+3UL,i) * v3;
639  C(j ,i4) += A(j ,i) * v4;
640  C(j+1UL,i4) += A(j+1UL,i) * v4;
641  C(j+2UL,i4) += A(j+2UL,i) * v4;
642  C(j+3UL,i4) += A(j+3UL,i) * v4;
643  }
644  for( ; j<A.rows(); ++j ) {
645  C(j,i1) += A(j,i) * v1;
646  C(j,i2) += A(j,i) * v2;
647  C(j,i3) += A(j,i) * v3;
648  C(j,i4) += A(j,i) * v4;
649  }
650  }
651 
652  for( ; element!=end; ++element ) {
653  for( j=0UL; j<jend; j+=4UL ) {
654  C(j ,element->index()) += A(j ,i) * element->value();
655  C(j+1UL,element->index()) += A(j+1UL,i) * element->value();
656  C(j+2UL,element->index()) += A(j+2UL,i) * element->value();
657  C(j+3UL,element->index()) += A(j+3UL,i) * element->value();
658  }
659  for( ; j<A.rows(); ++j ) {
660  C(j,element->index()) += A(j,i) * element->value();
661  }
662  }
663  }
664  }
666  //**********************************************************************************************
667 
668  //**Assignment to sparse matrices***************************************************************
681  template< typename MT // Type of the target sparse matrix
682  , bool SO > // Storage order of the target sparse matrix
683  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
684  assign( SparseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
685  {
687 
688  typedef typename SelectType< SO, ResultType, OppositeType >::Type TmpType;
689 
696 
697  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
698  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
699 
700  const TmpType tmp( serial( rhs ) );
701  assign( ~lhs, tmp );
702  }
704  //**********************************************************************************************
705 
706  //**Restructuring assignment to row-major matrices**********************************************
721  template< typename MT > // Type of the target matrix
722  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
723  assign( Matrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
724  {
726 
728 
729  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
730  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
731 
732  assign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
733  }
735  //**********************************************************************************************
736 
737  //**Restructuring assignment to column-major matrices*******************************************
752  template< typename MT > // Type of the target matrix
753  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
754  assign( Matrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
755  {
757 
758  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
759  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
760 
761  assign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
762  }
764  //**********************************************************************************************
765 
766  //**Addition assignment to row-major dense matrices*********************************************
779  template< typename MT > // Type of the target dense matrix
780  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
781  addAssign( DenseMatrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
782  {
784 
786 
787  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
788  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
789 
790  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
791  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
792 
793  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
794  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
795  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
796  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
797  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
798  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
799 
800  TDMatSMatMultExpr::selectRowMajorAddAssignKernel( ~lhs, A, B );
801  }
803  //**********************************************************************************************
804 
805  //**Optimized addition assignment to row-major dense matrices***********************************
819  template< typename MT3 // Type of the left-hand side target matrix
820  , typename MT4 // Type of the left-hand side matrix operand
821  , typename MT5 > // Type of the right-hand side matrix operand
822  static inline void selectRowMajorAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
823  {
824  typedef typename MT5::ConstIterator ConstIterator;
825 
826  const size_t last( A.rows() & size_t(-4) );
827  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == last, "Invalid end calculation" );
828 
829  for( size_t i=0UL; i<last; i+=4UL ) {
830  for( size_t j=0UL; j<B.rows(); ++j ) {
831  ConstIterator element( B.begin(j) );
832  const ConstIterator end( B.end(j) );
833  for( ; element!=end; ++element ) {
834  C(i ,element->index()) += A(i ,j) * element->value();
835  C(i+1UL,element->index()) += A(i+1UL,j) * element->value();
836  C(i+2UL,element->index()) += A(i+2UL,j) * element->value();
837  C(i+3UL,element->index()) += A(i+3UL,j) * element->value();
838  }
839  }
840  }
841 
842  for( size_t i=last; i<A.rows(); ++i ) {
843  for( size_t j=0UL; j<B.rows(); ++j ) {
844  ConstIterator element( B.begin(j) );
845  const ConstIterator end( B.end(j) );
846  for( ; element!=end; ++element ) {
847  C(i,element->index()) += A(i,j) * element->value();
848  }
849  }
850  }
851  }
853  //**********************************************************************************************
854 
855  //**Addition assignment to column-major dense matrices******************************************
868  template< typename MT > // Type of the target dense matrix
869  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
870  addAssign( DenseMatrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
871  {
873 
874  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
875  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
876 
877  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
878  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
879 
880  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
881  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
882  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
883  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
884  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
885  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
886 
887  TDMatSMatMultExpr::selectColumnMajorAddAssignKernel( ~lhs, A, B );
888  }
890  //**********************************************************************************************
891 
892  //**Optimized addition assignment to column-major dense matrices********************************
906  template< typename MT3 // Type of the left-hand side target matrix
907  , typename MT4 // Type of the left-hand side matrix operand
908  , typename MT5 > // Type of the right-hand side matrix operand
909  static inline void selectColumnMajorAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
910  {
911  typedef typename MT5::ConstIterator ConstIterator;
912 
913  const size_t jend( A.rows() & size_t(-4) );
914  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == jend, "Invalid end calculation" );
915  size_t j( 0UL );
916 
917  for( size_t i=0UL; i<B.rows(); ++i ) {
918  ConstIterator element( B.begin(i) );
919  const ConstIterator end( B.end(i) );
920 
921  const size_t nonzeros( B.nonZeros(i) );
922  const size_t kend( nonzeros & size_t(-4) );
923 
924  for( size_t k=0UL; k<kend; k+=4UL )
925  {
926  const size_t i1( element->index() );
927  const ET2 v1( element->value() );
928  ++element;
929  const size_t i2( element->index() );
930  const ET2 v2( element->value() );
931  ++element;
932  const size_t i3( element->index() );
933  const ET2 v3( element->value() );
934  ++element;
935  const size_t i4( element->index() );
936  const ET2 v4( element->value() );
937  ++element;
938 
939  for( j=0UL; j<jend; j+=4UL ) {
940  C(j ,i1) += A(j ,i) * v1;
941  C(j+1UL,i1) += A(j+1UL,i) * v1;
942  C(j+2UL,i1) += A(j+2UL,i) * v1;
943  C(j+3UL,i1) += A(j+3UL,i) * v1;
944  C(j ,i2) += A(j ,i) * v2;
945  C(j+1UL,i2) += A(j+1UL,i) * v2;
946  C(j+2UL,i2) += A(j+2UL,i) * v2;
947  C(j+3UL,i2) += A(j+3UL,i) * v2;
948  C(j ,i3) += A(j ,i) * v3;
949  C(j+1UL,i3) += A(j+1UL,i) * v3;
950  C(j+2UL,i3) += A(j+2UL,i) * v3;
951  C(j+3UL,i3) += A(j+3UL,i) * v3;
952  C(j ,i4) += A(j ,i) * v4;
953  C(j+1UL,i4) += A(j+1UL,i) * v4;
954  C(j+2UL,i4) += A(j+2UL,i) * v4;
955  C(j+3UL,i4) += A(j+3UL,i) * v4;
956  }
957  for( ; j<A.rows(); ++j ) {
958  C(j,i1) += A(j,i) * v1;
959  C(j,i2) += A(j,i) * v2;
960  C(j,i3) += A(j,i) * v3;
961  C(j,i4) += A(j,i) * v4;
962  }
963  }
964 
965  for( ; element!=end; ++element ) {
966  for( j=0UL; j<jend; j+=4UL ) {
967  C(j ,element->index()) += A(j ,i) * element->value();
968  C(j+1UL,element->index()) += A(j+1UL,i) * element->value();
969  C(j+2UL,element->index()) += A(j+2UL,i) * element->value();
970  C(j+3UL,element->index()) += A(j+3UL,i) * element->value();
971  }
972  for( ; j<A.rows(); ++j ) {
973  C(j,element->index()) += A(j,i) * element->value();
974  }
975  }
976  }
977  }
979  //**********************************************************************************************
980 
981  //**Restructuring addition assignment to row-major matrices*************************************
997  template< typename MT > // Type of the target matrix
998  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
999  addAssign( Matrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
1000  {
1002 
1004 
1005  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1006  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1007 
1008  addAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1009  }
1011  //**********************************************************************************************
1012 
1013  //**Restructuring addition assignment to column-major matrices**********************************
1028  template< typename MT > // Type of the target matrix
1029  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1030  addAssign( Matrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
1031  {
1033 
1034  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1035  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1036 
1037  addAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1038  }
1040  //**********************************************************************************************
1041 
1042  //**Addition assignment to sparse matrices******************************************************
1043  // No special implementation for the addition assignment to sparse matrices.
1044  //**********************************************************************************************
1045 
1046  //**Subtraction assignment to row-major dense matrices******************************************
1059  template< typename MT > // Type of the target dense matrix
1060  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1061  subAssign( DenseMatrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
1062  {
1064 
1066 
1067  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1068  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1069 
1070  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1071  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1072 
1073  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1074  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1075  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1076  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1077  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1078  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1079 
1080  TDMatSMatMultExpr::selectRowMajorSubAssignKernel( ~lhs, A, B );
1081  }
1083  //**********************************************************************************************
1084 
1085  //**Optimized subtraction assignment to row-major dense matrices********************************
1099  template< typename MT3 // Type of the left-hand side target matrix
1100  , typename MT4 // Type of the left-hand side matrix operand
1101  , typename MT5 > // Type of the right-hand side matrix operand
1102  static inline void selectRowMajorSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1103  {
1104  typedef typename MT5::ConstIterator ConstIterator;
1105 
1106  const size_t last( A.rows() & size_t(-4) );
1107  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == last, "Invalid end calculation" );
1108 
1109  for( size_t i=0UL; i<last; i+=4UL ) {
1110  for( size_t j=0UL; j<B.rows(); ++j ) {
1111  ConstIterator element( B.begin(j) );
1112  const ConstIterator end( B.end(j) );
1113  for( ; element!=end; ++element ) {
1114  C(i ,element->index()) -= A(i ,j) * element->value();
1115  C(i+1UL,element->index()) -= A(i+1UL,j) * element->value();
1116  C(i+2UL,element->index()) -= A(i+2UL,j) * element->value();
1117  C(i+3UL,element->index()) -= A(i+3UL,j) * element->value();
1118  }
1119  }
1120  }
1121 
1122  for( size_t i=last; i<A.rows(); ++i ) {
1123  for( size_t j=0UL; j<B.rows(); ++j ) {
1124  ConstIterator element( B.begin(j) );
1125  const ConstIterator end( B.end(j) );
1126  for( ; element!=end; ++element ) {
1127  C(i,element->index()) -= A(i,j) * element->value();
1128  }
1129  }
1130  }
1131  }
1133  //**********************************************************************************************
1134 
1135  //**Subtraction assignment to column-major dense matrices***************************************
1148  template< typename MT > // Type of the target dense matrix
1149  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1150  subAssign( DenseMatrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
1151  {
1153 
1154  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1155  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1156 
1157  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1158  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1159 
1160  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1161  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1162  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1163  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1164  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1165  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1166 
1167  TDMatSMatMultExpr::selectColumnMajorSubAssignKernel( ~lhs, A, B );
1168  }
1170  //**********************************************************************************************
1171 
1172  //**Optimized subtraction assignment to column-major dense matrices*****************************
1186  template< typename MT3 // Type of the left-hand side target matrix
1187  , typename MT4 // Type of the left-hand side matrix operand
1188  , typename MT5 > // Type of the right-hand side matrix operand
1189  static inline void selectColumnMajorSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1190  {
1191  typedef typename MT5::ConstIterator ConstIterator;
1192 
1193  const size_t jend( A.rows() & size_t(-4) );
1194  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == jend, "Invalid end calculation" );
1195  size_t j( 0UL );
1196 
1197  for( size_t i=0UL; i<B.rows(); ++i ) {
1198  ConstIterator element( B.begin(i) );
1199  const ConstIterator end( B.end(i) );
1200 
1201  const size_t nonzeros( B.nonZeros(i) );
1202  const size_t kend( nonzeros & size_t(-4) );
1203 
1204  for( size_t k=0UL; k<kend; k+=4UL )
1205  {
1206  const size_t i1( element->index() );
1207  const ET2 v1( element->value() );
1208  ++element;
1209  const size_t i2( element->index() );
1210  const ET2 v2( element->value() );
1211  ++element;
1212  const size_t i3( element->index() );
1213  const ET2 v3( element->value() );
1214  ++element;
1215  const size_t i4( element->index() );
1216  const ET2 v4( element->value() );
1217  ++element;
1218 
1219  for( j=0UL; j<jend; j+=4UL ) {
1220  C(j ,i1) -= A(j ,i) * v1;
1221  C(j+1UL,i1) -= A(j+1UL,i) * v1;
1222  C(j+2UL,i1) -= A(j+2UL,i) * v1;
1223  C(j+3UL,i1) -= A(j+3UL,i) * v1;
1224  C(j ,i2) -= A(j ,i) * v2;
1225  C(j+1UL,i2) -= A(j+1UL,i) * v2;
1226  C(j+2UL,i2) -= A(j+2UL,i) * v2;
1227  C(j+3UL,i2) -= A(j+3UL,i) * v2;
1228  C(j ,i3) -= A(j ,i) * v3;
1229  C(j+1UL,i3) -= A(j+1UL,i) * v3;
1230  C(j+2UL,i3) -= A(j+2UL,i) * v3;
1231  C(j+3UL,i3) -= A(j+3UL,i) * v3;
1232  C(j ,i4) -= A(j ,i) * v4;
1233  C(j+1UL,i4) -= A(j+1UL,i) * v4;
1234  C(j+2UL,i4) -= A(j+2UL,i) * v4;
1235  C(j+3UL,i4) -= A(j+3UL,i) * v4;
1236  }
1237  for( ; j<A.rows(); ++j ) {
1238  C(j,i1) -= A(j,i) * v1;
1239  C(j,i2) -= A(j,i) * v2;
1240  C(j,i3) -= A(j,i) * v3;
1241  C(j,i4) -= A(j,i) * v4;
1242  }
1243  }
1244 
1245  for( ; element!=end; ++element ) {
1246  for( j=0UL; j<jend; j+=4UL ) {
1247  C(j ,element->index()) -= A(j ,i) * element->value();
1248  C(j+1UL,element->index()) -= A(j+1UL,i) * element->value();
1249  C(j+2UL,element->index()) -= A(j+2UL,i) * element->value();
1250  C(j+3UL,element->index()) -= A(j+3UL,i) * element->value();
1251  }
1252  for( ; j<A.rows(); ++j ) {
1253  C(j,element->index()) -= A(j,i) * element->value();
1254  }
1255  }
1256  }
1257  }
1259  //**********************************************************************************************
1260 
1261  //**Restructuring subtraction assignment to row-major matrices**********************************
1277  template< typename MT > // Type of the target matrix
1278  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1279  subAssign( Matrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
1280  {
1282 
1283  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1284  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1285 
1286  subAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1287  }
1289  //**********************************************************************************************
1290 
1291  //**Restructuring subtraction assignment to column-major matrices*******************************
1306  template< typename MT > // Type of the target matrix
1307  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1308  subAssign( Matrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
1309  {
1311 
1313 
1314  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1315  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1316 
1317  subAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1318  }
1320  //**********************************************************************************************
1321 
1322  //**Subtraction assignment to sparse matrices***************************************************
1323  // No special implementation for the subtraction assignment to sparse matrices.
1324  //**********************************************************************************************
1325 
1326  //**Multiplication assignment to dense matrices*************************************************
1327  // No special implementation for the multiplication assignment to dense matrices.
1328  //**********************************************************************************************
1329 
1330  //**Multiplication assignment to sparse matrices************************************************
1331  // No special implementation for the multiplication assignment to sparse matrices.
1332  //**********************************************************************************************
1333 
1334  //**SMP assignment to dense matrices************************************************************
1350  template< typename MT // Type of the target dense matrix
1351  , bool SO > // Storage order of the target dense matrix
1352  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1353  smpAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1354  {
1356 
1357  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1358  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1359 
1360  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1361  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1362 
1363  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1364  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1365  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1366  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1367  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1368  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1369 
1370  smpAssign( ~lhs, A * B );
1371  }
1373  //**********************************************************************************************
1374 
1375  //**SMP assignment to sparse matrices***********************************************************
1391  template< typename MT // Type of the target sparse matrix
1392  , bool SO > // Storage order of the target sparse matrix
1393  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1394  smpAssign( SparseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1395  {
1397 
1398  typedef typename SelectType< SO, ResultType, OppositeType >::Type TmpType;
1399 
1406 
1407  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1408  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1409 
1410  const TmpType tmp( rhs );
1411  smpAssign( ~lhs, tmp );
1412  }
1414  //**********************************************************************************************
1415 
1416  //**Restructuring SMP assignment to row-major matrices******************************************
1431  template< typename MT > // Type of the target matrix
1432  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1433  smpAssign( Matrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
1434  {
1436 
1438 
1439  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1440  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1441 
1442  smpAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1443  }
1445  //**********************************************************************************************
1446 
1447  //**Restructuring SMP assignment to column-major matrices***************************************
1462  template< typename MT > // Type of the target matrix
1463  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1464  smpAssign( Matrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
1465  {
1467 
1468  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1469  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1470 
1471  smpAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1472  }
1474  //**********************************************************************************************
1475 
1476  //**SMP addition assignment to dense matrices***************************************************
1492  template< typename MT // Type of the target dense matrix
1493  , bool SO > // Storage order of the target sparse matrix
1494  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1495  smpAddAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1496  {
1498 
1499  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1500  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1501 
1502  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1503  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1504 
1505  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1506  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1507  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1508  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1509  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1510  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1511 
1512  smpAddAssign( ~lhs, A * B );
1513  }
1515  //**********************************************************************************************
1516 
1517  //**Restructuring SMP addition assignment to row-major matrices*********************************
1532  template< typename MT > // Type of the target matrix
1533  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1534  smpAddAssign( Matrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
1535  {
1537 
1538  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1539  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1540 
1541  smpAddAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1542  }
1544  //**********************************************************************************************
1545 
1546  //**Restructuring SMP addition assignment to column-major matrices******************************
1561  template< typename MT > // Type of the target matrix
1562  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1563  smpAddAssign( Matrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
1564  {
1566 
1568 
1569  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1570  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1571 
1572  smpAddAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1573  }
1575  //**********************************************************************************************
1576 
1577  //**SMP addition assignment to sparse matrices**************************************************
1578  // No special implementation for the SMP addition assignment to sparse matrices.
1579  //**********************************************************************************************
1580 
1581  //**SMP subtraction assignment to dense matrices************************************************
1597  template< typename MT // Type of the target dense matrix
1598  , bool SO > // Storage order of the target sparse matrix
1599  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1600  smpSubAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1601  {
1603 
1604  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1605  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1606 
1607  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1608  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1609 
1610  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1611  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1612  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1613  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1614  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1615  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1616 
1617  smpSubAssign( ~lhs, A * B );
1618  }
1620  //**********************************************************************************************
1621 
1622  //**Restructuring SMP subtraction assignment to row-major matrices******************************
1637  template< typename MT > // Type of the target matrix
1638  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1639  smpSubAssign( Matrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
1640  {
1642 
1644 
1645  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1646  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1647 
1648  smpSubAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1649  }
1651  //**********************************************************************************************
1652 
1653  //**Restructuring SMP subtraction assignment to column-major matrices***************************
1668  template< typename MT > // Type of the target matrix
1669  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1670  smpSubAssign( Matrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
1671  {
1673 
1674  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1675  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1676 
1677  smpSubAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1678  }
1680  //**********************************************************************************************
1681 
1682  //**SMP subtraction assignment to sparse matrices***********************************************
1683  // No special implementation for the SMP subtraction assignment to sparse matrices.
1684  //**********************************************************************************************
1685 
1686  //**SMP multiplication assignment to dense matrices*********************************************
1687  // No special implementation for the SMP multiplication assignment to dense matrices.
1688  //**********************************************************************************************
1689 
1690  //**SMP multiplication assignment to sparse matrices********************************************
1691  // No special implementation for the SMP multiplication assignment to sparse matrices.
1692  //**********************************************************************************************
1693 
1694  //**Compile time checks*************************************************************************
1702  //**********************************************************************************************
1703 };
1704 //*************************************************************************************************
1705 
1706 
1707 
1708 
1709 //=================================================================================================
1710 //
1711 // GLOBAL BINARY ARITHMETIC OPERATORS
1712 //
1713 //=================================================================================================
1714 
1715 //*************************************************************************************************
1745 template< typename T1 // Type of the left-hand side dense matrix
1746  , typename T2 > // Type of the right-hand side sparse matrix
1747 inline const TDMatSMatMultExpr<T1,T2>
1749 {
1751 
1752  if( (~lhs).columns() != (~rhs).rows() )
1753  throw std::invalid_argument( "Matrix sizes do not match" );
1754 
1755  return TDMatSMatMultExpr<T1,T2>( ~lhs, ~rhs );
1756 }
1757 //*************************************************************************************************
1758 
1759 
1760 
1761 
1762 //=================================================================================================
1763 //
1764 // ROWS SPECIALIZATIONS
1765 //
1766 //=================================================================================================
1767 
1768 //*************************************************************************************************
1770 template< typename MT1, typename MT2 >
1771 struct Rows< TDMatSMatMultExpr<MT1,MT2> >
1772  : public Rows<MT1>
1773 {};
1775 //*************************************************************************************************
1776 
1777 
1778 
1779 
1780 //=================================================================================================
1781 //
1782 // COLUMNS SPECIALIZATIONS
1783 //
1784 //=================================================================================================
1785 
1786 //*************************************************************************************************
1788 template< typename MT1, typename MT2 >
1789 struct Columns< TDMatSMatMultExpr<MT1,MT2> >
1790  : public Columns<MT2>
1791 {};
1793 //*************************************************************************************************
1794 
1795 
1796 
1797 
1798 //=================================================================================================
1799 //
1800 // ISLOWER SPECIALIZATIONS
1801 //
1802 //=================================================================================================
1803 
1804 //*************************************************************************************************
1806 template< typename MT1, typename MT2 >
1807 struct IsLower< TDMatSMatMultExpr<MT1,MT2> >
1808  : public IsTrue< IsLower<MT1>::value && IsLower<MT2>::value >
1809 {};
1811 //*************************************************************************************************
1812 
1813 
1814 
1815 
1816 //=================================================================================================
1817 //
1818 // ISUPPER SPECIALIZATIONS
1819 //
1820 //=================================================================================================
1821 
1822 //*************************************************************************************************
1824 template< typename MT1, typename MT2 >
1825 struct IsUpper< TDMatSMatMultExpr<MT1,MT2> >
1826  : public IsTrue< IsUpper<MT1>::value && IsUpper<MT2>::value >
1827 {};
1829 //*************************************************************************************************
1830 
1831 
1832 
1833 
1834 //=================================================================================================
1835 //
1836 // EXPRESSION TRAIT SPECIALIZATIONS
1837 //
1838 //=================================================================================================
1839 
1840 //*************************************************************************************************
1842 template< typename MT1, typename MT2, typename VT >
1843 struct TDMatDVecMultExprTrait< TDMatSMatMultExpr<MT1,MT2>, VT >
1844 {
1845  public:
1846  //**********************************************************************************************
1847  typedef typename SelectType< IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
1848  IsSparseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value &&
1849  IsDenseVector<VT>::value && IsColumnVector<VT>::value
1850  , typename TDMatDVecMultExprTrait< MT1, typename SMatDVecMultExprTrait<MT2,VT>::Type >::Type
1851  , INVALID_TYPE >::Type Type;
1852  //**********************************************************************************************
1853 };
1855 //*************************************************************************************************
1856 
1857 
1858 //*************************************************************************************************
1860 template< typename MT1, typename MT2, typename VT >
1861 struct TDMatSVecMultExprTrait< TDMatSMatMultExpr<MT1,MT2>, VT >
1862 {
1863  public:
1864  //**********************************************************************************************
1865  typedef typename SelectType< IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
1866  IsSparseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value &&
1867  IsSparseVector<VT>::value && IsColumnVector<VT>::value
1868  , typename TDMatSVecMultExprTrait< MT1, typename SMatSVecMultExprTrait<MT2,VT>::Type >::Type
1869  , INVALID_TYPE >::Type Type;
1870  //**********************************************************************************************
1871 };
1873 //*************************************************************************************************
1874 
1875 
1876 //*************************************************************************************************
1878 template< typename VT, typename MT1, typename MT2 >
1879 struct TDVecTDMatMultExprTrait< VT, TDMatSMatMultExpr<MT1,MT2> >
1880 {
1881  public:
1882  //**********************************************************************************************
1883  typedef typename SelectType< IsDenseVector<VT>::value && IsRowVector<VT>::value &&
1884  IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
1885  IsSparseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value
1886  , typename TDMatDVecMultExprTrait< MT1, typename SMatDVecMultExprTrait<MT2,VT>::Type >::Type
1887  , INVALID_TYPE >::Type Type;
1888  //**********************************************************************************************
1889 };
1891 //*************************************************************************************************
1892 
1893 
1894 //*************************************************************************************************
1896 template< typename VT, typename MT1, typename MT2 >
1897 struct TSVecTDMatMultExprTrait< VT, TDMatSMatMultExpr<MT1,MT2> >
1898 {
1899  public:
1900  //**********************************************************************************************
1901  typedef typename SelectType< IsSparseVector<VT>::value && IsRowVector<VT>::value &&
1902  IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
1903  IsSparseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value
1904  , typename TDVecSMatMultExprTrait< typename TSVecTDMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
1905  , INVALID_TYPE >::Type Type;
1906  //**********************************************************************************************
1907 };
1909 //*************************************************************************************************
1910 
1911 
1912 //*************************************************************************************************
1914 template< typename MT1, typename MT2, bool AF >
1915 struct SubmatrixExprTrait< TDMatSMatMultExpr<MT1,MT2>, AF >
1916 {
1917  public:
1918  //**********************************************************************************************
1919  typedef typename MultExprTrait< typename SubmatrixExprTrait<const MT1,AF>::Type
1920  , typename SubmatrixExprTrait<const MT2,AF>::Type >::Type Type;
1921  //**********************************************************************************************
1922 };
1924 //*************************************************************************************************
1925 
1926 
1927 //*************************************************************************************************
1929 template< typename MT1, typename MT2 >
1930 struct RowExprTrait< TDMatSMatMultExpr<MT1,MT2> >
1931 {
1932  public:
1933  //**********************************************************************************************
1934  typedef typename MultExprTrait< typename RowExprTrait<const MT1>::Type, MT2 >::Type Type;
1935  //**********************************************************************************************
1936 };
1938 //*************************************************************************************************
1939 
1940 
1941 //*************************************************************************************************
1943 template< typename MT1, typename MT2 >
1944 struct ColumnExprTrait< TDMatSMatMultExpr<MT1,MT2> >
1945 {
1946  public:
1947  //**********************************************************************************************
1948  typedef typename MultExprTrait< MT1, typename ColumnExprTrait<const MT2>::Type >::Type Type;
1949  //**********************************************************************************************
1950 };
1952 //*************************************************************************************************
1953 
1954 } // namespace blaze
1955 
1956 #endif
bool isAligned() const
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatSMatMultExpr.h:335
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:89
size_t columns() const
Returns the current number of columns of the matrix.
Definition: TDMatSMatMultExpr.h:281
Header file for the SMatDVecMultExprTrait class template.
ResultType::TransposeType TransposeType
Transpose type for expression template evaluations.
Definition: TDMatSMatMultExpr.h:198
Header file for the Rows type trait.
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:4838
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:258
Header file for the IsSparseMatrix type trait.
Efficient implementation of a compressed matrix.The CompressedMatrix class template is the represent...
Definition: CompressedMatrix.h:205
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:79
Header file for the ColumnExprTrait class template.
TDMatSMatMultExpr(const MT1 &lhs, const MT2 &rhs)
Constructor for the TDMatSMatMultExpr class.
Definition: TDMatSMatMultExpr.h:231
Header file for the IsColumnMajorMatrix type trait.
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:2478
Header file for the IsRowVector type trait.
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:257
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatSMatMultExpr.h:200
Header file for the TDVecSMatMultExprTrait class template.
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:695
Header file for the Computation base class.
SelectType< evaluateLeft, const RT1, CT1 >::Type LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatSMatMultExpr.h:210
Header file for the MatMatMultExpr base class.
Expression object for transpose dense matrix-sparse matrix multiplications.The TDMatSMatMultExpr clas...
Definition: Forward.h:128
size_t rows() const
Returns the current number of rows of the matrix.
Definition: TDMatSMatMultExpr.h:271
TDMatSMatMultExpr< MT1, MT2 > This
Type of this TDMatSMatMultExpr instance.
Definition: TDMatSMatMultExpr.h:195
ResultType::ElementType ElementType
Resulting element type.
Definition: TDMatSMatMultExpr.h:199
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:107
Constraint on the data type.
bool canAlias(const T *alias) const
Returns whether the expression can alias with the given address alias.
Definition: TDMatSMatMultExpr.h:313
Constraint on the data type.
bool isAliased(const T *alias) const
Returns whether the expression is aliased with the given address alias.
Definition: TDMatSMatMultExpr.h:325
Constraint on the data type.
Header file for the MultExprTrait class template.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:90
Compile time type selection.The SelectType class template selects one of the two given types T1 and T...
Definition: SelectType.h:59
Header file for the DisableIf class template.
Header file for the multiplication trait.
Header file for the IsSymmetric type trait.
MultTrait< RT1, RT2 >::Type ResultType
Result type for expression template evaluations.
Definition: TDMatSMatMultExpr.h:196
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: StorageOrder.h:161
Header file for the TSVecTDMatMultExprTrait class template.
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2482
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: TDMatSMatMultExpr.h:353
SelectType< IsExpression< MT2 >::value, const MT2, const MT2 & >::Type RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:207
Header file for the TDMatSVecMultExprTrait class template.
Header file for the DenseMatrix base class.
BLAZE_ALWAYS_INLINE void assign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the assignment of a matrix to a matrix.
Definition: Matrix.h:635
Header file for the Columns type trait.
Header file for the IsLower type trait.
RT2::ElementType ET2
Element type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:122
SelectType< IsExpression< MT1 >::value, const MT1, const MT1 & >::Type LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:204
Header file for the SMatSVecMultExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:78
LeftOperand leftOperand() const
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatSMatMultExpr.h:291
Constraints on the storage order of matrix types.
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:2476
Header file for the SelectType class template.
Header file for the RowExprTrait class template.
Header file for all forward declarations for expression class templates.
Header file for the IsDenseMatrix type trait.
Header file for the EnableIf class template.
Header file for the serial shim.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:165
ResultType::OppositeType OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: TDMatSMatMultExpr.h:197
EnableIf< IsDenseMatrix< MT1 > >::Type smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
Header file for the IsSparseVector type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type, a compilation error is created.
Definition: Symmetric.h:116
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: StorageOrder.h:81
Header file for run time assertion macros.
EnableIf< IsDenseMatrix< MT1 > >::Type smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Utility type for generic codes.
Base template for the MultTrait class.
Definition: MultTrait.h:142
BLAZE_ALWAYS_INLINE void addAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the addition assignment of a matrix to a matrix.
Definition: Matrix.h:742
Substitution Failure Is Not An Error (SFINAE) class.The EnableIf class template is an auxiliary tool ...
Definition: EnableIf.h:184
const size_t SMP_TDMATSMATMULT_THRESHOLD
SMP column-major dense matrix/row-major sparse matrix multiplication threshold.This threshold specifi...
Definition: Thresholds.h:972
Header file for the reset shim.
Header file for the isDefault shim.
BLAZE_ALWAYS_INLINE bool isDefault(const NonNumericProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: NonNumericProxy.h:874
Constraint on the data type.
BLAZE_ALWAYS_INLINE void reset(const NonNumericProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: NonNumericProxy.h:833
Substitution Failure Is Not An Error (SFINAE) class.The DisableIf class template is an auxiliary tool...
Definition: DisableIf.h:184
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:283
Header file for the IsDenseVector type trait.
SelectType< evaluateRight, const RT2, CT2 >::Type RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: TDMatSMatMultExpr.h:213
MT2::ResultType RT2
Result type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:120
MT1::CompositeType CT1
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:123
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatSMatMultExpr.h:352
Header file for the IsRowMajorMatrix type trait.
const DMatTransExpr< MT,!SO > trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:932
Header file for the IsComputation type trait class.
EnableIf< IsDenseMatrix< MT1 > >::Type smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
MT2::CompositeType CT2
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:124
Header file for the TDMatDVecMultExprTrait class template.
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
This ResultType
Result type for expression template evaluations.
Definition: CompressedMatrix.h:2473
Header file for the IsTrue value trait.
RT1::ElementType ET1
Element type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:121
Header file for basic type definitions.
Header file for the IsUpper type trait.
Header file for the IsColumnVector type trait.
Constraint on the data type.
Header file for the IsResizable type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the TDVecTDMatMultExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:79
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: TDMatSMatMultExpr.h:246
MT1::ResultType RT1
Result type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:119
RightOperand rightOperand() const
Returns the right-hand side sparse matrix operand.
Definition: TDMatSMatMultExpr.h:301
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatSMatMultExpr.h:201
Header file for the IsExpression type trait class.
Header file for the FunctionTrace class.
BLAZE_ALWAYS_INLINE void subAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the subtraction assignment of a matrix to matrix.
Definition: Matrix.h:849
bool canSMPAssign() const
Returns whether the expression can be used in SMP assignments.
Definition: TDMatSMatMultExpr.h:345