TDMatSMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATSMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATSMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <stdexcept>
54 #include <blaze/math/shims/Reset.h>
91 #include <blaze/util/Assert.h>
93 #include <blaze/util/DisableIf.h>
94 #include <blaze/util/EnableIf.h>
95 #include <blaze/util/InvalidType.h>
97 #include <blaze/util/mpl/And.h>
98 #include <blaze/util/mpl/Or.h>
99 #include <blaze/util/SelectType.h>
100 #include <blaze/util/Types.h>
102 
103 
104 namespace blaze {
105 
106 //=================================================================================================
107 //
108 // CLASS TDMATSMATMULTEXPR
109 //
110 //=================================================================================================
111 
112 //*************************************************************************************************
119 template< typename MT1 // Type of the left-hand side dense matrix
120  , typename MT2 > // Type of the right-hand side sparse matrix
121 class TDMatSMatMultExpr : public DenseMatrix< TDMatSMatMultExpr<MT1,MT2>, true >
122  , private MatMatMultExpr
123  , private Computation
124 {
125  private:
126  //**Type definitions****************************************************************************
127  typedef typename MT1::ResultType RT1;
128  typedef typename MT2::ResultType RT2;
129  typedef typename RT1::ElementType ET1;
130  typedef typename RT2::ElementType ET2;
131  typedef typename MT1::CompositeType CT1;
132  typedef typename MT2::CompositeType CT2;
133  //**********************************************************************************************
134 
135  //**********************************************************************************************
138  //**********************************************************************************************
139 
140  //**********************************************************************************************
142  enum { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
143  //**********************************************************************************************
144 
145  //**********************************************************************************************
147 
154  template< typename T1, typename T2, typename T3 >
155  struct CanExploitSymmetry {
156  enum { value = ( IsRowMajorMatrix<T1>::value && IsSymmetric<T2>::value ) ||
157  ( IsColumnMajorMatrix<T1>::value && IsSymmetric<T3>::value ) };
158  };
160  //**********************************************************************************************
161 
162  //**********************************************************************************************
164 
168  template< typename T1, typename T2, typename T3 >
169  struct IsEvaluationRequired {
170  enum { value = ( evaluateLeft || evaluateRight ) &&
171  !CanExploitSymmetry<T1,T2,T3>::value };
172  };
174  //**********************************************************************************************
175 
176  //**********************************************************************************************
178 
182  template< typename T1, typename T2, typename T3 >
183  struct UseOptimizedKernel {
184  enum { value = !IsDiagonal<T2>::value &&
185  !IsResizable<typename T1::ElementType>::value &&
186  !( IsColumnMajorMatrix<T1>::value && IsResizable<ET2>::value ) };
187  };
189  //**********************************************************************************************
190 
191  //**********************************************************************************************
193 
196  template< typename T1, typename T2, typename T3 >
197  struct UseDefaultKernel {
198  enum { value = !UseOptimizedKernel<T1,T2,T3>::value };
199  };
201  //**********************************************************************************************
202 
203  public:
204  //**Type definitions****************************************************************************
210  typedef const ElementType ReturnType;
211  typedef const ResultType CompositeType;
212 
214  typedef typename SelectType< IsExpression<MT1>::value, const MT1, const MT1& >::Type LeftOperand;
215 
217  typedef typename SelectType< IsExpression<MT2>::value, const MT2, const MT2& >::Type RightOperand;
218 
221 
224  //**********************************************************************************************
225 
226  //**Compilation flags***************************************************************************
228  enum { vectorizable = 0 };
229 
231  enum { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
232  !evaluateRight && MT2::smpAssignable };
233  //**********************************************************************************************
234 
235  //**Constructor*********************************************************************************
241  explicit inline TDMatSMatMultExpr( const MT1& lhs, const MT2& rhs )
242  : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
243  , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
244  {
245  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
246  }
247  //**********************************************************************************************
248 
249  //**Access operator*****************************************************************************
256  inline ReturnType operator()( size_t i, size_t j ) const {
257  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
258  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
259 
260  const size_t kbegin( ( IsUpper<MT1>::value )
261  ?( ( IsLower<MT2>::value )
262  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
263  , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
264  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
265  :( ( IsLower<MT2>::value )
266  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
267  :( 0UL ) ) );
268  const size_t kend( ( IsLower<MT1>::value )
269  ?( ( IsUpper<MT2>::value )
270  ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
271  , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
272  :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
273  :( ( IsUpper<MT2>::value )
274  ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
275  :( lhs_.columns() ) ) );
276 
277  if( lhs_.columns() == 0UL ||
278  ( ( IsTriangular<MT1>::value || IsTriangular<MT2>::value ) && kbegin >= kend ) )
279  return ElementType();
280 
281  ElementType tmp( lhs_(i,kbegin) * rhs_(kbegin,j) );
282  for( size_t k=kbegin+1UL; k<kend; ++k ) {
283  tmp += lhs_(i,k) * rhs_(k,j);
284  }
285 
286  return tmp;
287  }
288  //**********************************************************************************************
289 
290  //**Rows function*******************************************************************************
295  inline size_t rows() const {
296  return lhs_.rows();
297  }
298  //**********************************************************************************************
299 
300  //**Columns function****************************************************************************
305  inline size_t columns() const {
306  return rhs_.columns();
307  }
308  //**********************************************************************************************
309 
310  //**Left operand access*************************************************************************
315  inline LeftOperand leftOperand() const {
316  return lhs_;
317  }
318  //**********************************************************************************************
319 
320  //**Right operand access************************************************************************
325  inline RightOperand rightOperand() const {
326  return rhs_;
327  }
328  //**********************************************************************************************
329 
330  //**********************************************************************************************
336  template< typename T >
337  inline bool canAlias( const T* alias ) const {
338  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
339  }
340  //**********************************************************************************************
341 
342  //**********************************************************************************************
348  template< typename T >
349  inline bool isAliased( const T* alias ) const {
350  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
351  }
352  //**********************************************************************************************
353 
354  //**********************************************************************************************
359  inline bool isAligned() const {
360  return lhs_.isAligned();
361  }
362  //**********************************************************************************************
363 
364  //**********************************************************************************************
369  inline bool canSMPAssign() const {
370  return ( columns() > SMP_TDMATSMATMULT_THRESHOLD );
371  }
372  //**********************************************************************************************
373 
374  private:
375  //**Member variables****************************************************************************
376  LeftOperand lhs_;
377  RightOperand rhs_;
378  //**********************************************************************************************
379 
380  //**Assignment to row-major dense matrices******************************************************
393  template< typename MT > // Type of the target dense matrix
394  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
396  {
398 
400 
401  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
402  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
403 
404  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
405  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
406 
407  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
408  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
409  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
410  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
411  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
412  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
413 
414  TDMatSMatMultExpr::selectRowMajorAssignKernel( ~lhs, A, B );
415  }
417  //**********************************************************************************************
418 
419  //**Default assignment to row-major dense matrices**********************************************
434  template< typename MT3 // Type of the left-hand side target matrix
435  , typename MT4 // Type of the left-hand side matrix operand
436  , typename MT5 > // Type of the right-hand side matrix operand
437  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
438  selectRowMajorAssignKernel( MT3& C, const MT4& A, const MT5& B )
439  {
440  typedef typename MT5::ConstIterator ConstIterator;
441 
442  for( size_t i=0UL; i<A.rows(); ++i )
443  {
444  for( size_t j=0UL; j<C.columns(); ++j ) {
445  reset( C(i,j) );
446  }
447 
448  if( IsDiagonal<MT4>::value )
449  {
450  ConstIterator element( B.begin(i) );
451  const ConstIterator end( B.end(i) );
452 
453  for( ; element!=end; ++element ) {
454  C(i,element->index()) = A(i,i) * element->value();
455  }
456  }
457  else
458  {
459  const size_t jbegin( ( IsUpper<MT4>::value )
460  ?( IsStrictlyUpper<MT4>::value ? i+1UL : i )
461  :( 0UL ) );
462  const size_t jend( ( IsLower<MT4>::value )
463  ?( IsStrictlyLower<MT4>::value ? i : i+1UL )
464  :( B.rows() ) );
465  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
466 
467  for( size_t j=jbegin; j<jend; ++j )
468  {
469  ConstIterator element( B.begin(j) );
470  const ConstIterator end( B.end(j) );
471 
472  for( ; element!=end; ++element ) {
473  if( isDefault( C(i,element->index()) ) )
474  C(i,element->index()) = A(i,j) * element->value();
475  else
476  C(i,element->index()) += A(i,j) * element->value();
477  }
478  }
479  }
480  }
481  }
483  //**********************************************************************************************
484 
485  //**Optimized assignment to row-major dense matrices********************************************
500  template< typename MT3 // Type of the left-hand side target matrix
501  , typename MT4 // Type of the left-hand side matrix operand
502  , typename MT5 > // Type of the right-hand side matrix operand
503  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
504  selectRowMajorAssignKernel( MT3& C, const MT4& A, const MT5& B )
505  {
506  typedef typename MT5::ConstIterator ConstIterator;
507 
508  const size_t ipos( A.rows() & size_t(-4) );
509  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == ipos, "Invalid end calculation" );
510 
511  for( size_t i=0UL; i<ipos; i+=4 )
512  {
513  for( size_t j=0UL; j<C.columns(); ++j ) {
514  reset( C(i ,j) );
515  reset( C(i+1UL,j) );
516  reset( C(i+2UL,j) );
517  reset( C(i+3UL,j) );
518  }
519 
520  const size_t jbegin( ( IsUpper<MT4>::value )
521  ?( IsStrictlyUpper<MT4>::value ? i+1UL : i )
522  :( 0UL ) );
523  const size_t jend( ( IsLower<MT4>::value )
524  ?( IsStrictlyLower<MT4>::value ? i+3UL : i+4UL )
525  :( B.rows() ) );
526  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
527 
528  for( size_t j=jbegin; j<jend; ++j )
529  {
530  ConstIterator element( B.begin(j) );
531  const ConstIterator end( B.end(j) );
532 
533  for( ; element!=end; ++element ) {
534  C(i ,element->index()) += A(i ,j) * element->value();
535  C(i+1UL,element->index()) += A(i+1UL,j) * element->value();
536  C(i+2UL,element->index()) += A(i+2UL,j) * element->value();
537  C(i+3UL,element->index()) += A(i+3UL,j) * element->value();
538  }
539  }
540  }
541 
542  for( size_t i=ipos; i<A.rows(); ++i )
543  {
544  for( size_t j=0UL; j<C.columns(); ++j ) {
545  reset( C(i,j) );
546  }
547 
548  const size_t jbegin( ( IsUpper<MT4>::value )
549  ?( IsStrictlyUpper<MT4>::value ? i+1UL : i )
550  :( 0UL ) );
551  const size_t jend( ( IsLower<MT4>::value )
552  ?( IsStrictlyLower<MT4>::value ? i : i+1UL )
553  :( B.rows() ) );
554  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
555 
556  for( size_t j=jbegin; j<jend; ++j )
557  {
558  ConstIterator element( B.begin(j) );
559  const ConstIterator end( B.end(j) );
560 
561  for( ; element!=end; ++element ) {
562  C(i,element->index()) += A(i,j) * element->value();
563  }
564  }
565  }
566  }
568  //**********************************************************************************************
569 
570  //**Assignment to column-major dense matrices***************************************************
583  template< typename MT > // Type of the target dense matrix
584  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
585  assign( DenseMatrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
586  {
588 
589  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
590  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
591 
592  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
593  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
594 
595  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
596  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
597  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
598  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
599  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
600  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
601 
602  TDMatSMatMultExpr::selectColumnMajorAssignKernel( ~lhs, A, B );
603  }
605  //**********************************************************************************************
606 
607  //**Default assignment to column-major dense matrices*******************************************
622  template< typename MT3 // Type of the left-hand side target matrix
623  , typename MT4 // Type of the left-hand side matrix operand
624  , typename MT5 > // Type of the right-hand side matrix operand
625  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
626  selectColumnMajorAssignKernel( MT3& C, const MT4& A, const MT5& B )
627  {
628  typedef typename MT5::ConstIterator ConstIterator;
629 
630  reset( C );
631 
632  for( size_t j=0UL; j<B.rows(); ++j )
633  {
634  ConstIterator element( B.begin(j) );
635  const ConstIterator end( B.end(j) );
636 
637  if( IsDiagonal<MT4>::value )
638  {
639  for( ; element!=end; ++element ) {
640  C(j,element->index()) = A(j,j) * element->value();
641  }
642  }
643  else
644  {
645  const size_t ibegin( ( IsLower<MT4>::value )
646  ?( IsStrictlyLower<MT4>::value ? j+1UL : j )
647  :( 0UL ) );
648  const size_t iend( ( IsUpper<MT4>::value )
649  ?( IsStrictlyUpper<MT4>::value ? j : j+1UL )
650  :( A.rows() ) );
651  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
652 
653  for( ; element!=end; ++element ) {
654  for( size_t i=ibegin; i<iend; ++i ) {
655  if( isDefault( C(i,element->index()) ) )
656  C(i,element->index()) = A(i,j) * element->value();
657  else
658  C(i,element->index()) += A(i,j) * element->value();
659  }
660  }
661  }
662  }
663  }
665  //**********************************************************************************************
666 
667  //**Optimized assignment to column-major dense matrices*****************************************
682  template< typename MT3 // Type of the left-hand side target matrix
683  , typename MT4 // Type of the left-hand side matrix operand
684  , typename MT5 > // Type of the right-hand side matrix operand
685  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
686  selectColumnMajorAssignKernel( MT3& C, const MT4& A, const MT5& B )
687  {
688  typedef typename MT5::ConstIterator ConstIterator;
689 
690  reset( C );
691 
692  size_t i( 0UL );
693 
694  for( size_t j=0UL; j<B.rows(); ++j )
695  {
696  ConstIterator element( B.begin(j) );
697  const ConstIterator end( B.end(j) );
698 
699  const size_t nonzeros( B.nonZeros(j) );
700  const size_t kpos( nonzeros & size_t(-4) );
701  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
702 
703  const size_t ibegin( ( IsLower<MT4>::value )
704  ?( IsStrictlyLower<MT4>::value ? j+1UL : j )
705  :( 0UL ) );
706  const size_t iend( ( IsUpper<MT4>::value )
707  ?( IsStrictlyUpper<MT4>::value ? j : j+1UL )
708  :( A.rows() ) );
709  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
710 
711  const size_t inum( iend - ibegin );
712  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
713  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
714 
715  for( size_t k=0UL; k<kpos; k+=4UL )
716  {
717  const size_t j1( element->index() );
718  const ET2 v1( element->value() );
719  ++element;
720  const size_t j2( element->index() );
721  const ET2 v2( element->value() );
722  ++element;
723  const size_t j3( element->index() );
724  const ET2 v3( element->value() );
725  ++element;
726  const size_t j4( element->index() );
727  const ET2 v4( element->value() );
728  ++element;
729 
730  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
731 
732  for( i=ibegin; i<ipos; i+=4UL ) {
733  C(i ,j1) += A(i ,j) * v1;
734  C(i+1UL,j1) += A(i+1UL,j) * v1;
735  C(i+2UL,j1) += A(i+2UL,j) * v1;
736  C(i+3UL,j1) += A(i+3UL,j) * v1;
737  C(i ,j2) += A(i ,j) * v2;
738  C(i+1UL,j2) += A(i+1UL,j) * v2;
739  C(i+2UL,j2) += A(i+2UL,j) * v2;
740  C(i+3UL,j2) += A(i+3UL,j) * v2;
741  C(i ,j3) += A(i ,j) * v3;
742  C(i+1UL,j3) += A(i+1UL,j) * v3;
743  C(i+2UL,j3) += A(i+2UL,j) * v3;
744  C(i+3UL,j3) += A(i+3UL,j) * v3;
745  C(i ,j4) += A(i ,j) * v4;
746  C(i+1UL,j4) += A(i+1UL,j) * v4;
747  C(i+2UL,j4) += A(i+2UL,j) * v4;
748  C(i+3UL,j4) += A(i+3UL,j) * v4;
749  }
750  for( ; i<iend; ++i ) {
751  C(i,j1) += A(i,j) * v1;
752  C(i,j2) += A(i,j) * v2;
753  C(i,j3) += A(i,j) * v3;
754  C(i,j4) += A(i,j) * v4;
755  }
756  }
757 
758  for( ; element!=end; ++element )
759  {
760  const size_t j1( element->index() );
761  const ET2 v1( element->value() );
762 
763  for( i=ibegin; i<ipos; i+=4UL ) {
764  C(i ,j1) += A(i ,j) * v1;
765  C(i+1UL,j1) += A(i+1UL,j) * v1;
766  C(i+2UL,j1) += A(i+2UL,j) * v1;
767  C(i+3UL,j1) += A(i+3UL,j) * v1;
768  }
769  for( ; i<iend; ++i ) {
770  C(i,j1) += A(i,j) * v1;
771  }
772  }
773  }
774  }
776  //**********************************************************************************************
777 
778  //**Assignment to sparse matrices***************************************************************
791  template< typename MT // Type of the target sparse matrix
792  , bool SO > // Storage order of the target sparse matrix
793  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
794  assign( SparseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
795  {
797 
798  typedef typename SelectType< SO, ResultType, OppositeType >::Type TmpType;
799 
806 
807  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
808  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
809 
810  const TmpType tmp( serial( rhs ) );
811  assign( ~lhs, tmp );
812  }
814  //**********************************************************************************************
815 
816  //**Restructuring assignment to row-major matrices**********************************************
831  template< typename MT > // Type of the target matrix
832  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
833  assign( Matrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
834  {
836 
838 
839  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
840  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
841 
842  assign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
843  }
845  //**********************************************************************************************
846 
847  //**Restructuring assignment to column-major matrices*******************************************
862  template< typename MT > // Type of the target matrix
863  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
864  assign( Matrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
865  {
867 
868  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
869  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
870 
871  assign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
872  }
874  //**********************************************************************************************
875 
876  //**Addition assignment to row-major dense matrices*********************************************
889  template< typename MT > // Type of the target dense matrix
890  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
891  addAssign( DenseMatrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
892  {
894 
896 
897  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
898  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
899 
900  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
901  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
902 
903  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
904  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
905  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
906  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
907  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
908  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
909 
910  TDMatSMatMultExpr::selectRowMajorAddAssignKernel( ~lhs, A, B );
911  }
913  //**********************************************************************************************
914 
915  //**Optimized addition assignment to row-major dense matrices***********************************
929  template< typename MT3 // Type of the left-hand side target matrix
930  , typename MT4 // Type of the left-hand side matrix operand
931  , typename MT5 > // Type of the right-hand side matrix operand
932  static inline void selectRowMajorAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
933  {
934  typedef typename MT5::ConstIterator ConstIterator;
935 
936  size_t i( 0UL );
937 
938  if( !IsDiagonal<MT4>::value )
939  {
940  const size_t ipos( A.rows() & size_t(-4) );
941  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == ipos, "Invalid end calculation" );
942 
943  for( ; i<ipos; i+=4UL )
944  {
945  const size_t jbegin( ( IsUpper<MT4>::value )
946  ?( IsStrictlyUpper<MT4>::value ? i+1UL : i )
947  :( 0UL ) );
948  const size_t jend( ( IsLower<MT4>::value )
949  ?( IsStrictlyLower<MT4>::value ? i+3UL : i+4UL )
950  :( B.rows() ) );
951  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
952 
953  for( size_t j=jbegin; j<jend; ++j )
954  {
955  ConstIterator element( B.begin(j) );
956  const ConstIterator end( B.end(j) );
957 
958  for( ; element!=end; ++element ) {
959  C(i ,element->index()) += A(i ,j) * element->value();
960  C(i+1UL,element->index()) += A(i+1UL,j) * element->value();
961  C(i+2UL,element->index()) += A(i+2UL,j) * element->value();
962  C(i+3UL,element->index()) += A(i+3UL,j) * element->value();
963  }
964  }
965  }
966  }
967 
968  for( ; i<A.rows(); ++i )
969  {
970  if( IsDiagonal<MT4>::value )
971  {
972  ConstIterator element( B.begin(i) );
973  const ConstIterator end( B.end(i) );
974 
975  for( ; element!=end; ++element ) {
976  C(i,element->index()) += A(i,i) * element->value();
977  }
978  }
979  else
980  {
981  const size_t jbegin( ( IsUpper<MT4>::value )
982  ?( IsStrictlyUpper<MT4>::value ? i+1UL : i )
983  :( 0UL ) );
984  const size_t jend( ( IsLower<MT4>::value )
985  ?( IsStrictlyLower<MT4>::value ? i : i+1UL )
986  :( B.rows() ) );
987  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
988 
989  for( size_t j=jbegin; j<jend; ++j )
990  {
991  ConstIterator element( B.begin(j) );
992  const ConstIterator end( B.end(j) );
993 
994  for( ; element!=end; ++element ) {
995  C(i,element->index()) += A(i,j) * element->value();
996  }
997  }
998  }
999  }
1000  }
1002  //**********************************************************************************************
1003 
1004  //**Addition assignment to column-major dense matrices******************************************
1017  template< typename MT > // Type of the target dense matrix
1018  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1019  addAssign( DenseMatrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
1020  {
1022 
1023  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1024  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1025 
1026  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1027  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1028 
1029  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1030  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1031  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1032  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1033  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1034  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1035 
1036  TDMatSMatMultExpr::selectColumnMajorAddAssignKernel( ~lhs, A, B );
1037  }
1039  //**********************************************************************************************
1040 
1041  //**Default addition assignment to column-major dense matrices**********************************
1055  template< typename MT3 // Type of the left-hand side target matrix
1056  , typename MT4 // Type of the left-hand side matrix operand
1057  , typename MT5 > // Type of the right-hand side matrix operand
1058  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
1059  selectColumnMajorAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1060  {
1061  typedef typename MT5::ConstIterator ConstIterator;
1062 
1063  size_t i( 0UL );
1064 
1065  for( size_t j=0UL; j<B.rows(); ++j )
1066  {
1067  ConstIterator element( B.begin(j) );
1068  const ConstIterator end( B.end(j) );
1069 
1070  if( IsDiagonal<MT4>::value )
1071  {
1072  for( ; element!=end; ++element ) {
1073  C(j,element->index()) += A(j,j) * element->value();
1074  }
1075  }
1076  else
1077  {
1078  const size_t ibegin( ( IsLower<MT4>::value )
1079  ?( IsStrictlyLower<MT4>::value ? j+1UL : j )
1080  :( 0UL ) );
1081  const size_t iend( ( IsUpper<MT4>::value )
1082  ?( IsStrictlyUpper<MT4>::value ? j : j+1UL )
1083  :( A.rows() ) );
1084  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1085 
1086  const size_t inum( iend - ibegin );
1087  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1088  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1089 
1090  for( ; element!=end; ++element ) {
1091  for( i=ibegin; i<ipos; i+=4UL ) {
1092  C(i ,element->index()) += A(i ,j) * element->value();
1093  C(i+1UL,element->index()) += A(i+1UL,j) * element->value();
1094  C(i+2UL,element->index()) += A(i+2UL,j) * element->value();
1095  C(i+3UL,element->index()) += A(i+3UL,j) * element->value();
1096  }
1097  for( ; i<iend; ++i ) {
1098  C(i,element->index()) += A(i,j) * element->value();
1099  }
1100  }
1101  }
1102  }
1103  }
1105  //**********************************************************************************************
1106 
1107  //**Optimized addition assignment to column-major dense matrices********************************
1121  template< typename MT3 // Type of the left-hand side target matrix
1122  , typename MT4 // Type of the left-hand side matrix operand
1123  , typename MT5 > // Type of the right-hand side matrix operand
1124  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
1125  selectColumnMajorAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1126  {
1127  typedef typename MT5::ConstIterator ConstIterator;
1128 
1129  size_t i( 0UL );
1130 
1131  for( size_t j=0UL; j<B.rows(); ++j )
1132  {
1133  ConstIterator element( B.begin(j) );
1134  const ConstIterator end( B.end(j) );
1135 
1136  const size_t nonzeros( B.nonZeros(j) );
1137  const size_t kpos( nonzeros & size_t(-4) );
1138  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1139 
1140  const size_t ibegin( ( IsLower<MT4>::value )
1141  ?( IsStrictlyLower<MT4>::value ? j+1UL : j )
1142  :( 0UL ) );
1143  const size_t iend( ( IsUpper<MT4>::value )
1144  ?( IsStrictlyUpper<MT4>::value ? j : j+1UL )
1145  :( A.rows() ) );
1146  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1147 
1148  const size_t inum( iend - ibegin );
1149  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1150  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1151 
1152  for( size_t k=0UL; k<kpos; k+=4UL )
1153  {
1154  const size_t j1( element->index() );
1155  const ET2 v1( element->value() );
1156  ++element;
1157  const size_t j2( element->index() );
1158  const ET2 v2( element->value() );
1159  ++element;
1160  const size_t j3( element->index() );
1161  const ET2 v3( element->value() );
1162  ++element;
1163  const size_t j4( element->index() );
1164  const ET2 v4( element->value() );
1165  ++element;
1166 
1167  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1168 
1169  for( i=ibegin; i<ipos; i+=4UL ) {
1170  C(i ,j1) += A(i ,j) * v1;
1171  C(i+1UL,j1) += A(i+1UL,j) * v1;
1172  C(i+2UL,j1) += A(i+2UL,j) * v1;
1173  C(i+3UL,j1) += A(i+3UL,j) * v1;
1174  C(i ,j2) += A(i ,j) * v2;
1175  C(i+1UL,j2) += A(i+1UL,j) * v2;
1176  C(i+2UL,j2) += A(i+2UL,j) * v2;
1177  C(i+3UL,j2) += A(i+3UL,j) * v2;
1178  C(i ,j3) += A(i ,j) * v3;
1179  C(i+1UL,j3) += A(i+1UL,j) * v3;
1180  C(i+2UL,j3) += A(i+2UL,j) * v3;
1181  C(i+3UL,j3) += A(i+3UL,j) * v3;
1182  C(i ,j4) += A(i ,j) * v4;
1183  C(i+1UL,j4) += A(i+1UL,j) * v4;
1184  C(i+2UL,j4) += A(i+2UL,j) * v4;
1185  C(i+3UL,j4) += A(i+3UL,j) * v4;
1186  }
1187  for( ; i<iend; ++i ) {
1188  C(i,j1) += A(i,j) * v1;
1189  C(i,j2) += A(i,j) * v2;
1190  C(i,j3) += A(i,j) * v3;
1191  C(i,j4) += A(i,j) * v4;
1192  }
1193  }
1194 
1195  for( ; element!=end; ++element )
1196  {
1197  const size_t j1( element->index() );
1198  const ET2 v1( element->value() );
1199 
1200  for( i=ibegin; i<ipos; i+=4UL ) {
1201  C(i ,j1) += A(i ,j) * v1;
1202  C(i+1UL,j1) += A(i+1UL,j) * v1;
1203  C(i+2UL,j1) += A(i+2UL,j) * v1;
1204  C(i+3UL,j1) += A(i+3UL,j) * v1;
1205  }
1206  for( ; i<iend; ++i ) {
1207  C(i,j1) += A(i,j) * v1;
1208  }
1209  }
1210  }
1211  }
1213  //**********************************************************************************************
1214 
1215  //**Restructuring addition assignment to row-major matrices*************************************
1231  template< typename MT > // Type of the target matrix
1232  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1233  addAssign( Matrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
1234  {
1236 
1238 
1239  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1240  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1241 
1242  addAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1243  }
1245  //**********************************************************************************************
1246 
1247  //**Restructuring addition assignment to column-major matrices**********************************
1262  template< typename MT > // Type of the target matrix
1263  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1264  addAssign( Matrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
1265  {
1267 
1268  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1269  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1270 
1271  addAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1272  }
1274  //**********************************************************************************************
1275 
1276  //**Addition assignment to sparse matrices******************************************************
1277  // No special implementation for the addition assignment to sparse matrices.
1278  //**********************************************************************************************
1279 
1280  //**Subtraction assignment to row-major dense matrices******************************************
1293  template< typename MT > // Type of the target dense matrix
1294  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1295  subAssign( DenseMatrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
1296  {
1298 
1300 
1301  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1302  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1303 
1304  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1305  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1306 
1307  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1308  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1309  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1310  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1311  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1312  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1313 
1314  TDMatSMatMultExpr::selectRowMajorSubAssignKernel( ~lhs, A, B );
1315  }
1317  //**********************************************************************************************
1318 
1319  //**Optimized subtraction assignment to row-major dense matrices********************************
1333  template< typename MT3 // Type of the left-hand side target matrix
1334  , typename MT4 // Type of the left-hand side matrix operand
1335  , typename MT5 > // Type of the right-hand side matrix operand
1336  static inline void selectRowMajorSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1337  {
1338  typedef typename MT5::ConstIterator ConstIterator;
1339 
1340  size_t i( 0UL );
1341 
1342  if( !IsDiagonal<MT4>::value )
1343  {
1344  const size_t ipos( A.rows() & size_t(-4) );
1345  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == ipos, "Invalid end calculation" );
1346 
1347  for( ; i<ipos; i+=4UL )
1348  {
1349  const size_t jbegin( ( IsUpper<MT4>::value )
1350  ?( IsStrictlyUpper<MT4>::value ? i+1UL : i )
1351  :( 0UL ) );
1352  const size_t jend( ( IsLower<MT4>::value )
1353  ?( IsStrictlyLower<MT4>::value ? i+3UL : i+4UL )
1354  :( B.rows() ) );
1355  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1356 
1357  for( size_t j=jbegin; j<jend; ++j )
1358  {
1359  ConstIterator element( B.begin(j) );
1360  const ConstIterator end( B.end(j) );
1361 
1362  for( ; element!=end; ++element ) {
1363  C(i ,element->index()) -= A(i ,j) * element->value();
1364  C(i+1UL,element->index()) -= A(i+1UL,j) * element->value();
1365  C(i+2UL,element->index()) -= A(i+2UL,j) * element->value();
1366  C(i+3UL,element->index()) -= A(i+3UL,j) * element->value();
1367  }
1368  }
1369  }
1370  }
1371 
1372  for( ; i<A.rows(); ++i )
1373  {
1374  if( IsDiagonal<MT4>::value )
1375  {
1376  ConstIterator element( B.begin(i) );
1377  const ConstIterator end( B.end(i) );
1378 
1379  for( ; element!=end; ++element ) {
1380  C(i,element->index()) -= A(i,i) * element->value();
1381  }
1382  }
1383  else
1384  {
1385  const size_t jbegin( ( IsUpper<MT4>::value )
1386  ?( IsStrictlyUpper<MT4>::value ? i+1UL : i )
1387  :( 0UL ) );
1388  const size_t jend( ( IsLower<MT4>::value )
1389  ?( IsStrictlyLower<MT4>::value ? i : i+1UL )
1390  :( B.rows() ) );
1391  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1392 
1393  for( size_t j=jbegin; j<jend; ++j )
1394  {
1395  ConstIterator element( B.begin(j) );
1396  const ConstIterator end( B.end(j) );
1397 
1398  for( ; element!=end; ++element ) {
1399  C(i,element->index()) -= A(i,j) * element->value();
1400  }
1401  }
1402  }
1403  }
1404  }
1406  //**********************************************************************************************
1407 
1408  //**Subtraction assignment to column-major dense matrices***************************************
1421  template< typename MT > // Type of the target dense matrix
1422  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1423  subAssign( DenseMatrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
1424  {
1426 
1427  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1428  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1429 
1430  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1431  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1432 
1433  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1434  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1435  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1436  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1437  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1438  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1439 
1440  TDMatSMatMultExpr::selectColumnMajorSubAssignKernel( ~lhs, A, B );
1441  }
1443  //**********************************************************************************************
1444 
1445  //**Default subtraction assignment to column-major dense matrices*******************************
1459  template< typename MT3 // Type of the left-hand side target matrix
1460  , typename MT4 // Type of the left-hand side matrix operand
1461  , typename MT5 > // Type of the right-hand side matrix operand
1462  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
1463  selectColumnMajorSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1464  {
1465  typedef typename MT5::ConstIterator ConstIterator;
1466 
1467  size_t i( 0UL );
1468 
1469  for( size_t j=0UL; j<B.rows(); ++j )
1470  {
1471  ConstIterator element( B.begin(j) );
1472  const ConstIterator end( B.end(j) );
1473 
1474  if( IsDiagonal<MT4>::value )
1475  {
1476  for( ; element!=end; ++element ) {
1477  C(j,element->index()) -= A(j,j) * element->value();
1478  }
1479  }
1480  else
1481  {
1482  const size_t ibegin( ( IsLower<MT4>::value )
1483  ?( IsStrictlyLower<MT4>::value ? j+1UL : j )
1484  :( 0UL ) );
1485  const size_t iend( ( IsUpper<MT4>::value )
1486  ?( IsStrictlyUpper<MT4>::value ? j : j+1UL )
1487  :( A.rows() ) );
1488  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1489 
1490  const size_t inum( iend - ibegin );
1491  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1492  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1493 
1494  for( ; element!=end; ++element ) {
1495  for( i=ibegin; i<ipos; i+=4UL ) {
1496  C(i ,element->index()) -= A(i ,j) * element->value();
1497  C(i+1UL,element->index()) -= A(i+1UL,j) * element->value();
1498  C(i+2UL,element->index()) -= A(i+2UL,j) * element->value();
1499  C(i+3UL,element->index()) -= A(i+3UL,j) * element->value();
1500  }
1501  for( ; i<iend; ++i ) {
1502  C(i,element->index()) -= A(i,j) * element->value();
1503  }
1504  }
1505  }
1506  }
1507  }
1509  //**********************************************************************************************
1510 
1511  //**Optimized subtraction assignment to column-major dense matrices*****************************
1525  template< typename MT3 // Type of the left-hand side target matrix
1526  , typename MT4 // Type of the left-hand side matrix operand
1527  , typename MT5 > // Type of the right-hand side matrix operand
1528  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
1529  selectColumnMajorSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1530  {
1531  typedef typename MT5::ConstIterator ConstIterator;
1532 
1533  size_t i( 0UL );
1534 
1535  for( size_t j=0UL; j<B.rows(); ++j )
1536  {
1537  ConstIterator element( B.begin(j) );
1538  const ConstIterator end( B.end(j) );
1539 
1540  const size_t nonzeros( B.nonZeros(j) );
1541  const size_t kpos( nonzeros & size_t(-4) );
1542  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1543 
1544  const size_t ibegin( ( IsLower<MT4>::value )
1545  ?( IsStrictlyLower<MT4>::value ? j+1UL : j )
1546  :( 0UL ) );
1547  const size_t iend( ( IsUpper<MT4>::value )
1548  ?( IsStrictlyUpper<MT4>::value ? j : j+1UL )
1549  :( A.rows() ) );
1550  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1551 
1552  const size_t inum( iend - ibegin );
1553  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1554  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1555 
1556  for( size_t k=0UL; k<kpos; k+=4UL )
1557  {
1558  const size_t j1( element->index() );
1559  const ET2 v1( element->value() );
1560  ++element;
1561  const size_t j2( element->index() );
1562  const ET2 v2( element->value() );
1563  ++element;
1564  const size_t j3( element->index() );
1565  const ET2 v3( element->value() );
1566  ++element;
1567  const size_t j4( element->index() );
1568  const ET2 v4( element->value() );
1569  ++element;
1570 
1571  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1572 
1573  for( i=ibegin; i<ipos; i+=4UL ) {
1574  C(i ,j1) -= A(i ,j) * v1;
1575  C(i+1UL,j1) -= A(i+1UL,j) * v1;
1576  C(i+2UL,j1) -= A(i+2UL,j) * v1;
1577  C(i+3UL,j1) -= A(i+3UL,j) * v1;
1578  C(i ,j2) -= A(i ,j) * v2;
1579  C(i+1UL,j2) -= A(i+1UL,j) * v2;
1580  C(i+2UL,j2) -= A(i+2UL,j) * v2;
1581  C(i+3UL,j2) -= A(i+3UL,j) * v2;
1582  C(i ,j3) -= A(i ,j) * v3;
1583  C(i+1UL,j3) -= A(i+1UL,j) * v3;
1584  C(i+2UL,j3) -= A(i+2UL,j) * v3;
1585  C(i+3UL,j3) -= A(i+3UL,j) * v3;
1586  C(i ,j4) -= A(i ,j) * v4;
1587  C(i+1UL,j4) -= A(i+1UL,j) * v4;
1588  C(i+2UL,j4) -= A(i+2UL,j) * v4;
1589  C(i+3UL,j4) -= A(i+3UL,j) * v4;
1590  }
1591  for( ; i<iend; ++i ) {
1592  C(i,j1) -= A(i,j) * v1;
1593  C(i,j2) -= A(i,j) * v2;
1594  C(i,j3) -= A(i,j) * v3;
1595  C(i,j4) -= A(i,j) * v4;
1596  }
1597  }
1598 
1599  for( ; element!=end; ++element )
1600  {
1601  const size_t j1( element->index() );
1602  const ET2 v1( element->value() );
1603 
1604  for( i=ibegin; i<ipos; i+=4UL ) {
1605  C(i ,j1) -= A(i ,j) * v1;
1606  C(i+1UL,j1) -= A(i+1UL,j) * v1;
1607  C(i+2UL,j1) -= A(i+2UL,j) * v1;
1608  C(i+3UL,j1) -= A(i+3UL,j) * v1;
1609  }
1610  for( ; i<iend; ++i ) {
1611  C(i,j1) -= A(i,j) * v1;
1612  }
1613  }
1614  }
1615  }
1617  //**********************************************************************************************
1618 
1619  //**Restructuring subtraction assignment to row-major matrices**********************************
1635  template< typename MT > // Type of the target matrix
1636  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1637  subAssign( Matrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
1638  {
1640 
1641  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1642  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1643 
1644  subAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1645  }
1647  //**********************************************************************************************
1648 
1649  //**Restructuring subtraction assignment to column-major matrices*******************************
1664  template< typename MT > // Type of the target matrix
1665  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1666  subAssign( Matrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
1667  {
1669 
1671 
1672  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1673  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1674 
1675  subAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1676  }
1678  //**********************************************************************************************
1679 
1680  //**Subtraction assignment to sparse matrices***************************************************
1681  // No special implementation for the subtraction assignment to sparse matrices.
1682  //**********************************************************************************************
1683 
1684  //**Multiplication assignment to dense matrices*************************************************
1685  // No special implementation for the multiplication assignment to dense matrices.
1686  //**********************************************************************************************
1687 
1688  //**Multiplication assignment to sparse matrices************************************************
1689  // No special implementation for the multiplication assignment to sparse matrices.
1690  //**********************************************************************************************
1691 
1692  //**SMP assignment to dense matrices************************************************************
1708  template< typename MT // Type of the target dense matrix
1709  , bool SO > // Storage order of the target dense matrix
1710  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1711  smpAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1712  {
1714 
1715  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1716  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1717 
1718  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1719  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1720 
1721  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1722  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1723  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1724  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1725  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1726  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1727 
1728  smpAssign( ~lhs, A * B );
1729  }
1731  //**********************************************************************************************
1732 
1733  //**SMP assignment to sparse matrices***********************************************************
1749  template< typename MT // Type of the target sparse matrix
1750  , bool SO > // Storage order of the target sparse matrix
1751  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1752  smpAssign( SparseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1753  {
1755 
1756  typedef typename SelectType< SO, ResultType, OppositeType >::Type TmpType;
1757 
1764 
1765  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1766  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1767 
1768  const TmpType tmp( rhs );
1769  smpAssign( ~lhs, tmp );
1770  }
1772  //**********************************************************************************************
1773 
1774  //**Restructuring SMP assignment to row-major matrices******************************************
1789  template< typename MT > // Type of the target matrix
1790  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1791  smpAssign( Matrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
1792  {
1794 
1796 
1797  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1798  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1799 
1800  smpAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1801  }
1803  //**********************************************************************************************
1804 
1805  //**Restructuring SMP assignment to column-major matrices***************************************
1820  template< typename MT > // Type of the target matrix
1821  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1822  smpAssign( Matrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
1823  {
1825 
1826  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1827  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1828 
1829  smpAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1830  }
1832  //**********************************************************************************************
1833 
1834  //**SMP addition assignment to dense matrices***************************************************
1850  template< typename MT // Type of the target dense matrix
1851  , bool SO > // Storage order of the target sparse matrix
1852  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1853  smpAddAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1854  {
1856 
1857  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1858  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1859 
1860  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1861  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1862 
1863  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1864  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1865  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1866  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1867  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1868  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1869 
1870  smpAddAssign( ~lhs, A * B );
1871  }
1873  //**********************************************************************************************
1874 
1875  //**Restructuring SMP addition assignment to row-major matrices*********************************
1890  template< typename MT > // Type of the target matrix
1891  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1892  smpAddAssign( Matrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
1893  {
1895 
1896  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1897  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1898 
1899  smpAddAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1900  }
1902  //**********************************************************************************************
1903 
1904  //**Restructuring SMP addition assignment to column-major matrices******************************
1919  template< typename MT > // Type of the target matrix
1920  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1921  smpAddAssign( Matrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
1922  {
1924 
1926 
1927  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1928  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1929 
1930  smpAddAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1931  }
1933  //**********************************************************************************************
1934 
1935  //**SMP addition assignment to sparse matrices**************************************************
1936  // No special implementation for the SMP addition assignment to sparse matrices.
1937  //**********************************************************************************************
1938 
1939  //**SMP subtraction assignment to dense matrices************************************************
1955  template< typename MT // Type of the target dense matrix
1956  , bool SO > // Storage order of the target sparse matrix
1957  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1958  smpSubAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1959  {
1961 
1962  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1963  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1964 
1965  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1966  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1967 
1968  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1969  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1970  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1971  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1972  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1973  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1974 
1975  smpSubAssign( ~lhs, A * B );
1976  }
1978  //**********************************************************************************************
1979 
1980  //**Restructuring SMP subtraction assignment to row-major matrices******************************
1995  template< typename MT > // Type of the target matrix
1996  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1997  smpSubAssign( Matrix<MT,false>& lhs, const TDMatSMatMultExpr& rhs )
1998  {
2000 
2002 
2003  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
2004  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
2005 
2006  smpSubAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
2007  }
2009  //**********************************************************************************************
2010 
2011  //**Restructuring SMP subtraction assignment to column-major matrices***************************
2026  template< typename MT > // Type of the target matrix
2027  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
2028  smpSubAssign( Matrix<MT,true>& lhs, const TDMatSMatMultExpr& rhs )
2029  {
2031 
2032  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
2033  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
2034 
2035  smpSubAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
2036  }
2038  //**********************************************************************************************
2039 
2040  //**SMP subtraction assignment to sparse matrices***********************************************
2041  // No special implementation for the SMP subtraction assignment to sparse matrices.
2042  //**********************************************************************************************
2043 
2044  //**SMP multiplication assignment to dense matrices*********************************************
2045  // No special implementation for the SMP multiplication assignment to dense matrices.
2046  //**********************************************************************************************
2047 
2048  //**SMP multiplication assignment to sparse matrices********************************************
2049  // No special implementation for the SMP multiplication assignment to sparse matrices.
2050  //**********************************************************************************************
2051 
2052  //**Compile time checks*************************************************************************
2060  //**********************************************************************************************
2061 };
2062 //*************************************************************************************************
2063 
2064 
2065 
2066 
2067 //=================================================================================================
2068 //
2069 // GLOBAL BINARY ARITHMETIC OPERATORS
2070 //
2071 //=================================================================================================
2072 
2073 //*************************************************************************************************
2103 template< typename T1 // Type of the left-hand side dense matrix
2104  , typename T2 > // Type of the right-hand side sparse matrix
2105 inline const TDMatSMatMultExpr<T1,T2>
2107 {
2109 
2110  if( (~lhs).columns() != (~rhs).rows() )
2111  throw std::invalid_argument( "Matrix sizes do not match" );
2112 
2113  return TDMatSMatMultExpr<T1,T2>( ~lhs, ~rhs );
2114 }
2115 //*************************************************************************************************
2116 
2117 
2118 
2119 
2120 //=================================================================================================
2121 //
2122 // ROWS SPECIALIZATIONS
2123 //
2124 //=================================================================================================
2125 
2126 //*************************************************************************************************
2128 template< typename MT1, typename MT2 >
2129 struct Rows< TDMatSMatMultExpr<MT1,MT2> >
2130  : public Rows<MT1>
2131 {};
2133 //*************************************************************************************************
2134 
2135 
2136 
2137 
2138 //=================================================================================================
2139 //
2140 // COLUMNS SPECIALIZATIONS
2141 //
2142 //=================================================================================================
2143 
2144 //*************************************************************************************************
2146 template< typename MT1, typename MT2 >
2147 struct Columns< TDMatSMatMultExpr<MT1,MT2> >
2148  : public Columns<MT2>
2149 {};
2151 //*************************************************************************************************
2152 
2153 
2154 
2155 
2156 //=================================================================================================
2157 //
2158 // ISLOWER SPECIALIZATIONS
2159 //
2160 //=================================================================================================
2161 
2162 //*************************************************************************************************
2164 template< typename MT1, typename MT2 >
2165 struct IsLower< TDMatSMatMultExpr<MT1,MT2> >
2166  : public IsTrue< And< IsLower<MT1>, IsLower<MT2> >::value >
2167 {};
2169 //*************************************************************************************************
2170 
2171 
2172 
2173 
2174 //=================================================================================================
2175 //
2176 // ISUNILOWER SPECIALIZATIONS
2177 //
2178 //=================================================================================================
2179 
2180 //*************************************************************************************************
2182 template< typename MT1, typename MT2 >
2183 struct IsUniLower< TDMatSMatMultExpr<MT1,MT2> >
2184  : public IsTrue< And< IsUniLower<MT1>, IsUniLower<MT2> >::value >
2185 {};
2187 //*************************************************************************************************
2188 
2189 
2190 
2191 
2192 //=================================================================================================
2193 //
2194 // ISSTRICTLYLOWER SPECIALIZATIONS
2195 //
2196 //=================================================================================================
2197 
2198 //*************************************************************************************************
2200 template< typename MT1, typename MT2 >
2201 struct IsStrictlyLower< TDMatSMatMultExpr<MT1,MT2> >
2202  : public IsTrue< Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2203  , And< IsStrictlyLower<MT2>, IsLower<MT1> > >::value >
2204 {};
2206 //*************************************************************************************************
2207 
2208 
2209 
2210 
2211 //=================================================================================================
2212 //
2213 // ISUPPER SPECIALIZATIONS
2214 //
2215 //=================================================================================================
2216 
2217 //*************************************************************************************************
2219 template< typename MT1, typename MT2 >
2220 struct IsUpper< TDMatSMatMultExpr<MT1,MT2> >
2221  : public IsTrue< And< IsUpper<MT1>, IsUpper<MT2> >::value >
2222 {};
2224 //*************************************************************************************************
2225 
2226 
2227 
2228 
2229 //=================================================================================================
2230 //
2231 // ISUNIUPPER SPECIALIZATIONS
2232 //
2233 //=================================================================================================
2234 
2235 //*************************************************************************************************
2237 template< typename MT1, typename MT2 >
2238 struct IsUniUpper< TDMatSMatMultExpr<MT1,MT2> >
2239  : public IsTrue< And< IsUniUpper<MT1>, IsUniUpper<MT2> >::value >
2240 {};
2242 //*************************************************************************************************
2243 
2244 
2245 
2246 
2247 //=================================================================================================
2248 //
2249 // ISSTRICTLYUPPER SPECIALIZATIONS
2250 //
2251 //=================================================================================================
2252 
2253 //*************************************************************************************************
2255 template< typename MT1, typename MT2 >
2256 struct IsStrictlyUpper< TDMatSMatMultExpr<MT1,MT2> >
2257  : public IsTrue< Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2258  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > >::value >
2259 {};
2261 //*************************************************************************************************
2262 
2263 
2264 
2265 
2266 //=================================================================================================
2267 //
2268 // EXPRESSION TRAIT SPECIALIZATIONS
2269 //
2270 //=================================================================================================
2271 
2272 //*************************************************************************************************
2274 template< typename MT1, typename MT2, typename VT >
2275 struct TDMatDVecMultExprTrait< TDMatSMatMultExpr<MT1,MT2>, VT >
2276 {
2277  public:
2278  //**********************************************************************************************
2279  typedef typename SelectType< IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
2280  IsSparseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value &&
2281  IsDenseVector<VT>::value && IsColumnVector<VT>::value
2282  , typename TDMatDVecMultExprTrait< MT1, typename SMatDVecMultExprTrait<MT2,VT>::Type >::Type
2283  , INVALID_TYPE >::Type Type;
2284  //**********************************************************************************************
2285 };
2287 //*************************************************************************************************
2288 
2289 
2290 //*************************************************************************************************
2292 template< typename MT1, typename MT2, typename VT >
2293 struct TDMatSVecMultExprTrait< TDMatSMatMultExpr<MT1,MT2>, VT >
2294 {
2295  public:
2296  //**********************************************************************************************
2297  typedef typename SelectType< IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
2298  IsSparseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value &&
2299  IsSparseVector<VT>::value && IsColumnVector<VT>::value
2300  , typename TDMatSVecMultExprTrait< MT1, typename SMatSVecMultExprTrait<MT2,VT>::Type >::Type
2301  , INVALID_TYPE >::Type Type;
2302  //**********************************************************************************************
2303 };
2305 //*************************************************************************************************
2306 
2307 
2308 //*************************************************************************************************
2310 template< typename VT, typename MT1, typename MT2 >
2311 struct TDVecTDMatMultExprTrait< VT, TDMatSMatMultExpr<MT1,MT2> >
2312 {
2313  public:
2314  //**********************************************************************************************
2315  typedef typename SelectType< IsDenseVector<VT>::value && IsRowVector<VT>::value &&
2316  IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
2317  IsSparseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value
2318  , typename TDMatDVecMultExprTrait< MT1, typename SMatDVecMultExprTrait<MT2,VT>::Type >::Type
2319  , INVALID_TYPE >::Type Type;
2320  //**********************************************************************************************
2321 };
2323 //*************************************************************************************************
2324 
2325 
2326 //*************************************************************************************************
2328 template< typename VT, typename MT1, typename MT2 >
2329 struct TSVecTDMatMultExprTrait< VT, TDMatSMatMultExpr<MT1,MT2> >
2330 {
2331  public:
2332  //**********************************************************************************************
2333  typedef typename SelectType< IsSparseVector<VT>::value && IsRowVector<VT>::value &&
2334  IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
2335  IsSparseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value
2336  , typename TDVecSMatMultExprTrait< typename TSVecTDMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
2337  , INVALID_TYPE >::Type Type;
2338  //**********************************************************************************************
2339 };
2341 //*************************************************************************************************
2342 
2343 
2344 //*************************************************************************************************
2346 template< typename MT1, typename MT2, bool AF >
2347 struct SubmatrixExprTrait< TDMatSMatMultExpr<MT1,MT2>, AF >
2348 {
2349  public:
2350  //**********************************************************************************************
2351  typedef typename MultExprTrait< typename SubmatrixExprTrait<const MT1,AF>::Type
2352  , typename SubmatrixExprTrait<const MT2,AF>::Type >::Type Type;
2353  //**********************************************************************************************
2354 };
2356 //*************************************************************************************************
2357 
2358 
2359 //*************************************************************************************************
2361 template< typename MT1, typename MT2 >
2362 struct RowExprTrait< TDMatSMatMultExpr<MT1,MT2> >
2363 {
2364  public:
2365  //**********************************************************************************************
2366  typedef typename MultExprTrait< typename RowExprTrait<const MT1>::Type, MT2 >::Type Type;
2367  //**********************************************************************************************
2368 };
2370 //*************************************************************************************************
2371 
2372 
2373 //*************************************************************************************************
2375 template< typename MT1, typename MT2 >
2376 struct ColumnExprTrait< TDMatSMatMultExpr<MT1,MT2> >
2377 {
2378  public:
2379  //**********************************************************************************************
2380  typedef typename MultExprTrait< MT1, typename ColumnExprTrait<const MT2>::Type >::Type Type;
2381  //**********************************************************************************************
2382 };
2384 //*************************************************************************************************
2385 
2386 } // namespace blaze
2387 
2388 #endif
SelectType< evaluateLeft, const RT1, CT1 >::Type LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatSMatMultExpr.h:220
const MT::ElementType max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1649
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:89
MT1::ResultType RT1
Result type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:127
Header file for the SMatDVecMultExprTrait class template.
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatSMatMultExpr.h:211
Header file for the Rows type trait.
Header file for the IsUniUpper type trait.
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:8247
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:105
Header file for basic type definitions.
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:258
Header file for the IsSparseMatrix type trait.
Efficient implementation of a compressed matrix.The CompressedMatrix class template is the represent...
Definition: CompressedMatrix.h:209
Header file for the IsDiagonal type trait.
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:79
Header file for the ColumnExprTrait class template.
Header file for the IsColumnMajorMatrix type trait.
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:821
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:2507
Header file for the IsRowVector type trait.
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:261
Header file for the And class template.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:90
size_t columns() const
Returns the current number of columns of the matrix.
Definition: TDMatSMatMultExpr.h:305
Header file for the TDVecSMatMultExprTrait class template.
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:699
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:90
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatSMatMultExpr.h:210
Expression object for transpose dense matrix-sparse matrix multiplications.The TDMatSMatMultExpr clas...
Definition: Forward.h:128
Header file for the IsUniLower type trait.
LeftOperand leftOperand() const
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatSMatMultExpr.h:315
bool isAligned() const
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatSMatMultExpr.h:359
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:107
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:861
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Header file for the MultExprTrait class template.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:90
bool canSMPAssign() const
Returns whether the expression can be used in SMP assignments.
Definition: TDMatSMatMultExpr.h:369
RT2::ElementType ET2
Element type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:130
Compile time type selection.The SelectType class template selects one of the two given types T1 and T...
Definition: SelectType.h:59
Header file for the DisableIf class template.
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
SelectType< IsExpression< MT2 >::value, const MT2, const MT2 & >::Type RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:217
Header file for the IsSymmetric type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: StorageOrder.h:161
Header file for the TSVecTDMatMultExprTrait class template.
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2511
Header file for the Or class template.
Header file for the TDMatSVecMultExprTrait class template.
const MT::ElementType min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1602
Header file for the DenseMatrix base class.
BLAZE_ALWAYS_INLINE void assign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the assignment of a matrix to a matrix.
Definition: Matrix.h:635
Header file for the Columns type trait.
MT2::CompositeType CT2
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:132
Header file for the IsLower type trait.
ResultType::ElementType ElementType
Resulting element type.
Definition: TDMatSMatMultExpr.h:209
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: TDMatSMatMultExpr.h:377
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatSMatMultExpr.h:376
Header file for the SMatSVecMultExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:78
Header file for the IsTriangular type trait.
Constraints on the storage order of matrix types.
bool isAliased(const T *alias) const
Returns whether the expression is aliased with the given address alias.
Definition: TDMatSMatMultExpr.h:349
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:2505
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: TDMatSMatMultExpr.h:256
ResultType::OppositeType OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: TDMatSMatMultExpr.h:207
Header file for the SelectType class template.
Header file for the RowExprTrait class template.
Header file for all forward declarations for expression class templates.
Header file for the IsDenseMatrix type trait.
MultTrait< RT1, RT2 >::Type ResultType
Result type for expression template evaluations.
Definition: TDMatSMatMultExpr.h:206
TDMatSMatMultExpr(const MT1 &lhs, const MT2 &rhs)
Constructor for the TDMatSMatMultExpr class.
Definition: TDMatSMatMultExpr.h:241
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
Header file for the serial shim.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:165
EnableIf< IsDenseMatrix< MT1 > >::Type smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
Header file for the IsSparseVector type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type, a compilation error is created.
Definition: Symmetric.h:116
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: StorageOrder.h:81
ResultType::TransposeType TransposeType
Transpose type for expression template evaluations.
Definition: TDMatSMatMultExpr.h:208
bool canAlias(const T *alias) const
Returns whether the expression can alias with the given address alias.
Definition: TDMatSMatMultExpr.h:337
Header file for run time assertion macros.
EnableIf< IsDenseMatrix< MT1 > >::Type smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Utility type for generic codes.
Base template for the MultTrait class.
Definition: MultTrait.h:150
BLAZE_ALWAYS_INLINE void addAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the addition assignment of a matrix to a matrix.
Definition: Matrix.h:742
SelectType< IsExpression< MT1 >::value, const MT1, const MT1 & >::Type LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:214
Substitution Failure Is Not An Error (SFINAE) class.The EnableIf class template is an auxiliary tool ...
Definition: EnableIf.h:184
const size_t SMP_TDMATSMATMULT_THRESHOLD
SMP column-major dense matrix/row-major sparse matrix multiplication threshold.This threshold specifi...
Definition: Thresholds.h:972
Header file for the reset shim.
Header file for the isDefault shim.
Constraint on the data type.
Substitution Failure Is Not An Error (SFINAE) class.The DisableIf class template is an auxiliary tool...
Definition: DisableIf.h:184
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:283
Header file for the IsDenseVector type trait.
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
MT2::ResultType RT2
Result type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:128
RightOperand rightOperand() const
Returns the right-hand side sparse matrix operand.
Definition: TDMatSMatMultExpr.h:325
Header file for the IsRowMajorMatrix type trait.
size_t rows() const
Returns the current number of rows of the matrix.
Definition: TDMatSMatMultExpr.h:295
const DMatTransExpr< MT,!SO > trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:937
TDMatSMatMultExpr< MT1, MT2 > This
Type of this TDMatSMatMultExpr instance.
Definition: TDMatSMatMultExpr.h:205
Header file for the IsComputation type trait class.
EnableIf< IsDenseMatrix< MT1 > >::Type smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
Header file for the TDMatDVecMultExprTrait class template.
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
This ResultType
Result type for expression template evaluations.
Definition: CompressedMatrix.h:2502
Header file for the IsTrue value trait.
MT1::CompositeType CT1
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:131
RT1::ElementType ET1
Element type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:129
SelectType< evaluateRight, const RT2, CT2 >::Type RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: TDMatSMatMultExpr.h:223
Header file for the IsUpper type trait.
Header file for the IsColumnVector type trait.
Constraint on the data type.
Header file for the IsResizable type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the TDVecTDMatMultExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:79
Header file for the IsExpression type trait class.
Header file for the FunctionTrace class.
BLAZE_ALWAYS_INLINE void subAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the subtraction assignment of a matrix to matrix.
Definition: Matrix.h:849