All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
SMatDMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <stdexcept>
52 #include <blaze/math/Intrinsics.h>
53 #include <blaze/math/shims/Reset.h>
84 #include <blaze/util/Assert.h>
85 #include <blaze/util/DisableIf.h>
86 #include <blaze/util/EnableIf.h>
87 #include <blaze/util/InvalidType.h>
89 #include <blaze/util/SelectType.h>
90 #include <blaze/util/Types.h>
94 
95 
96 namespace blaze {
97 
98 //=================================================================================================
99 //
100 // CLASS SMATDMATMULTEXPR
101 //
102 //=================================================================================================
103 
104 //*************************************************************************************************
111 template< typename MT1 // Type of the left-hand side sparse matrix
112  , typename MT2 > // Type of the right-hand side dense matrix
113 class SMatDMatMultExpr : public DenseMatrix< SMatDMatMultExpr<MT1,MT2>, false >
114  , private MatMatMultExpr
115  , private Computation
116 {
117  private:
118  //**Type definitions****************************************************************************
119  typedef typename MT1::ResultType RT1;
120  typedef typename MT2::ResultType RT2;
121  typedef typename RT1::ElementType ET1;
122  typedef typename RT2::ElementType ET2;
123  typedef typename MT1::CompositeType CT1;
124  typedef typename MT2::CompositeType CT2;
125  //**********************************************************************************************
126 
127  //**********************************************************************************************
130  //**********************************************************************************************
131 
132  //**********************************************************************************************
134  enum { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
135  //**********************************************************************************************
136 
137  //**********************************************************************************************
139 
145  template< typename T1, typename T2, typename T3 >
146  struct CanExploitSymmetry {
147  enum { value = IsColumnMajorMatrix<T1>::value &&
148  ( IsSymmetric<T2>::value || IsSymmetric<T3>::value ) };
149  };
151  //**********************************************************************************************
152 
153  //**********************************************************************************************
155 
159  template< typename T1, typename T2, typename T3 >
160  struct IsEvaluationRequired {
161  enum { value = ( evaluateLeft || evaluateRight ) &&
162  !CanExploitSymmetry<T1,T2,T3>::value };
163  };
165  //**********************************************************************************************
166 
167  //**********************************************************************************************
169 
172  template< typename T1, typename T2, typename T3 >
173  struct UseVectorizedKernel {
174  enum { value = T1::vectorizable && T3::vectorizable &&
175  IsRowMajorMatrix<T1>::value &&
176  IsSame<typename T1::ElementType,typename T2::ElementType>::value &&
177  IsSame<typename T1::ElementType,typename T3::ElementType>::value &&
178  IntrinsicTrait<typename T1::ElementType>::addition &&
179  IntrinsicTrait<typename T1::ElementType>::subtraction &&
180  IntrinsicTrait<typename T1::ElementType>::multiplication };
181  };
183  //**********************************************************************************************
184 
185  //**********************************************************************************************
187 
191  template< typename T1, typename T2, typename T3 >
192  struct UseOptimizedKernel {
193  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
194  !IsResizable<typename T1::ElementType>::value &&
195  !IsResizable<ET1>::value };
196  };
198  //**********************************************************************************************
199 
200  //**********************************************************************************************
202 
205  template< typename T1, typename T2, typename T3 >
206  struct UseDefaultKernel {
207  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
208  !UseOptimizedKernel<T1,T2,T3>::value };
209  };
211  //**********************************************************************************************
212 
213  public:
214  //**Type definitions****************************************************************************
221  typedef const ElementType ReturnType;
222  typedef const ResultType CompositeType;
223 
225  typedef typename SelectType< IsExpression<MT1>::value, const MT1, const MT1& >::Type LeftOperand;
226 
228  typedef typename SelectType< IsExpression<MT2>::value, const MT2, const MT2& >::Type RightOperand;
229 
232 
235  //**********************************************************************************************
236 
237  //**Compilation flags***************************************************************************
239  enum { vectorizable = MT2::vectorizable &&
243 
245  enum { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
246  !evaluateRight && MT2::smpAssignable };
247  //**********************************************************************************************
248 
249  //**Constructor*********************************************************************************
255  explicit inline SMatDMatMultExpr( const MT1& lhs, const MT2& rhs )
256  : lhs_( lhs ) // Left-hand side sparse matrix of the multiplication expression
257  , rhs_( rhs ) // Right-hand side dense matrix of the multiplication expression
258  {
259  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
260  }
261  //**********************************************************************************************
262 
263  //**Access operator*****************************************************************************
270  inline ReturnType operator()( size_t i, size_t j ) const {
271  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
272  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
273 
275 
276  ElementType tmp = ElementType();
277 
278  // Early exit
279  if( lhs_.columns() == 0 )
280  return tmp;
281 
282  // Fast computation in case the left-hand side sparse matrix directly provides iterators
284  {
285  CT1 A( lhs_ ); // Evaluation of the left-hand side sparse matrix operand
286 
287  const ConstIterator end( A.end(i) );
288  ConstIterator element( A.begin(i) );
289 
290  // Early exit in case row i is empty
291  if( element == end )
292  return tmp;
293 
294  // Calculating element (i,j)
295  tmp = element->value() * rhs_(element->index(),j);
296  ++element;
297  for( ; element!=end; ++element )
298  tmp += element->value() * rhs_(element->index(),j);
299  }
300 
301  // Default computation in case the left-hand side sparse matrix doesn't provide iterators
302  else {
303  tmp = lhs_(i,0) * rhs_(0,j);
304  for( size_t k=1; k<lhs_.columns(); ++k ) {
305  tmp += lhs_(i,k) * rhs_(k,j);
306  }
307  }
308 
309  return tmp;
310  }
311  //**********************************************************************************************
312 
313  //**Rows function*******************************************************************************
318  inline size_t rows() const {
319  return lhs_.rows();
320  }
321  //**********************************************************************************************
322 
323  //**Columns function****************************************************************************
328  inline size_t columns() const {
329  return rhs_.columns();
330  }
331  //**********************************************************************************************
332 
333  //**Left operand access*************************************************************************
338  inline LeftOperand leftOperand() const {
339  return lhs_;
340  }
341  //**********************************************************************************************
342 
343  //**Right operand access************************************************************************
348  inline RightOperand rightOperand() const {
349  return rhs_;
350  }
351  //**********************************************************************************************
352 
353  //**********************************************************************************************
359  template< typename T >
360  inline bool canAlias( const T* alias ) const {
361  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
362  }
363  //**********************************************************************************************
364 
365  //**********************************************************************************************
371  template< typename T >
372  inline bool isAliased( const T* alias ) const {
373  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
374  }
375  //**********************************************************************************************
376 
377  //**********************************************************************************************
382  inline bool isAligned() const {
383  return rhs_.isAligned();
384  }
385  //**********************************************************************************************
386 
387  //**********************************************************************************************
392  inline bool canSMPAssign() const {
393  return ( rows() > SMP_SMATDMATMULT_THRESHOLD );
394  }
395  //**********************************************************************************************
396 
397  private:
398  //**Member variables****************************************************************************
401  //**********************************************************************************************
402 
403  //**Assignment to dense matrices****************************************************************
416  template< typename MT // Type of the target dense matrix
417  , bool SO > // Storage order of the target dense matrix
418  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
419  assign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
420  {
422 
423  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
424  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
425 
426  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
427  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
428 
429  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
430  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
431  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
432  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
433  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
434  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
435 
436  SMatDMatMultExpr::selectAssignKernel( ~lhs, A, B );
437  }
439  //**********************************************************************************************
440 
441  //**Default assignment to row-major dense matrices**********************************************
455  template< typename MT3 // Type of the left-hand side target matrix
456  , typename MT4 // Type of the left-hand side matrix operand
457  , typename MT5 > // Type of the right-hand side matrix operand
458  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
459  selectAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
460  {
461  typedef typename MT4::ConstIterator ConstIterator;
462 
463  for( size_t i=0UL; i<A.rows(); ++i )
464  {
465  const ConstIterator end( A.end(i) );
466 
467  for( size_t j=0UL; j<B.columns(); ++j )
468  {
469  ConstIterator element( A.begin(i) );
470 
471  if( element != end ) {
472  (~C)(i,j) = element->value() * B(element->index(),j);
473  ++element;
474  for( ; element!=end; ++element ) {
475  (~C)(i,j) += element->value() * B(element->index(),j);
476  }
477  }
478  else {
479  reset( (~C)(i,j) );
480  }
481  }
482  }
483  }
485  //**********************************************************************************************
486 
487  //**Optimized assignment to row-major dense matrices********************************************
501  template< typename MT3 // Type of the left-hand side target matrix
502  , typename MT4 // Type of the left-hand side matrix operand
503  , typename MT5 > // Type of the right-hand side matrix operand
504  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
505  selectAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
506  {
507  typedef typename MT4::ConstIterator ConstIterator;
508 
509  const size_t jend( B.columns() & size_t(-4) );
510  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jend, "Invalid end calculation" );
511 
512  reset( ~C );
513 
514  for( size_t i=0UL; i<A.rows(); ++i )
515  {
516  const ConstIterator end( A.end(i) );
517  ConstIterator element( A.begin(i) );
518 
519  const size_t kend( A.nonZeros(i) & size_t(-4) );
520 
521  for( size_t k=0UL; k<kend; k+=4UL ) {
522  const size_t i1( element->index() );
523  const ET1 v1( element->value() );
524  ++element;
525  const size_t i2( element->index() );
526  const ET1 v2( element->value() );
527  ++element;
528  const size_t i3( element->index() );
529  const ET1 v3( element->value() );
530  ++element;
531  const size_t i4( element->index() );
532  const ET1 v4( element->value() );
533  ++element;
534 
535  for( size_t j=0UL; j<jend; j+=4UL ) {
536  (~C)(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
537  (~C)(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
538  (~C)(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
539  (~C)(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
540  }
541  for( size_t j=jend; j<B.columns(); ++j ) {
542  (~C)(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
543  }
544  }
545 
546  for( ; element!=end; ++element ) {
547  for( size_t j=0UL; j<jend; j+=4UL ) {
548  (~C)(i,j ) += element->value() * B(element->index(),j );
549  (~C)(i,j+1UL) += element->value() * B(element->index(),j+1UL);
550  (~C)(i,j+2UL) += element->value() * B(element->index(),j+2UL);
551  (~C)(i,j+3UL) += element->value() * B(element->index(),j+3UL);
552  }
553  for( size_t j=jend; j<B.columns(); ++j ) {
554  (~C)(i,j) += element->value() * B(element->index(),j);
555  }
556  }
557  }
558  }
560  //**********************************************************************************************
561 
562  //**Vectorized assignment to row-major dense matrices*******************************************
576  template< typename MT3 // Type of the left-hand side target matrix
577  , typename MT4 // Type of the left-hand side matrix operand
578  , typename MT5 > // Type of the right-hand side matrix operand
579  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
580  selectAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
581  {
582  typedef IntrinsicTrait<ElementType> IT;
583  typedef typename MT4::ConstIterator ConstIterator;
584 
585  const size_t N( B.columns() );
586 
587  reset( ~C );
588 
589  for( size_t i=0UL; i<A.rows(); ++i )
590  {
591  const ConstIterator end( A.end(i) );
592  ConstIterator element( A.begin(i) );
593 
594  const size_t kend( A.nonZeros(i) & size_t(-4) );
595 
596  for( size_t k=0UL; k<kend; k+=4UL ) {
597  const size_t i1( element->index() );
598  const IntrinsicType v1( set( element->value() ) );
599  ++element;
600  const size_t i2( element->index() );
601  const IntrinsicType v2( set( element->value() ) );
602  ++element;
603  const size_t i3( element->index() );
604  const IntrinsicType v3( set( element->value() ) );
605  ++element;
606  const size_t i4( element->index() );
607  const IntrinsicType v4( set( element->value() ) );
608  ++element;
609 
610  for( size_t j=0UL; j<N; j+=IT::size ) {
611  (~C).store( i, j, (~C).load(i,j) + v1 * B.load(i1,j) + v2 * B.load(i2,j) + v3 * B.load(i3,j) + v4 * B.load(i4,j) );
612  }
613  }
614 
615  for( ; element!=end; ++element ) {
616  const size_t i1( element->index() );
617  const IntrinsicType v1( set( element->value() ) );
618 
619  for( size_t j=0UL; j<N; j+=IT::size ) {
620  (~C).store( i, j, (~C).load(i,j) + v1 * B.load(i1,j) );
621  }
622  }
623  }
624  }
626  //**********************************************************************************************
627 
628  //**Default assignment to column-major dense matrices*******************************************
642  template< typename MT3 // Type of the left-hand side target matrix
643  , typename MT4 // Type of the left-hand side matrix operand
644  , typename MT5 > // Type of the right-hand side matrix operand
645  static inline void selectAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
646  {
647  typedef typename MT4::ConstIterator ConstIterator;
648 
649  const size_t jend( B.columns() & size_t(-4) );
650  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jend, "Invalid end calculation" );
651 
652  for( size_t j=0UL; j<jend; j+=4UL ) {
653  for( size_t i=0UL; i<A.rows(); ++i )
654  {
655  ConstIterator element( A.begin(i) );
656  const ConstIterator end( A.end(i) );
657 
658  if( element == end ) {
659  reset( (~C)(i,j ) );
660  reset( (~C)(i,j+1UL) );
661  reset( (~C)(i,j+2UL) );
662  reset( (~C)(i,j+3UL) );
663  continue;
664  }
665 
666  (~C)(i,j ) = element->value() * B(element->index(),j );
667  (~C)(i,j+1UL) = element->value() * B(element->index(),j+1UL);
668  (~C)(i,j+2UL) = element->value() * B(element->index(),j+2UL);
669  (~C)(i,j+3UL) = element->value() * B(element->index(),j+3UL);
670  ++element;
671  for( ; element!=end; ++element ) {
672  (~C)(i,j ) += element->value() * B(element->index(),j );
673  (~C)(i,j+1UL) += element->value() * B(element->index(),j+1UL);
674  (~C)(i,j+2UL) += element->value() * B(element->index(),j+2UL);
675  (~C)(i,j+3UL) += element->value() * B(element->index(),j+3UL);
676  }
677  }
678  }
679 
680  for( size_t j=jend; j<B.columns(); ++j ) {
681  for( size_t i=0UL; i<A.rows(); ++i )
682  {
683  ConstIterator element( A.begin(i) );
684  const ConstIterator end( A.end(i) );
685 
686  if( element == end ) {
687  reset( (~C)(i,j) );
688  continue;
689  }
690 
691  (~C)(i,j) = element->value() * B(element->index(),j);
692  ++element;
693  for( ; element!=end; ++element )
694  (~C)(i,j) += element->value() * B(element->index(),j);
695  }
696  }
697  }
699  //**********************************************************************************************
700 
701  //**Assignment to sparse matrices***************************************************************
714  template< typename MT // Type of the target sparse matrix
715  , bool SO > // Storage order of the target sparse matrix
716  friend inline void assign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
717  {
719 
720  typedef typename SelectType< SO, OppositeType, ResultType >::Type TmpType;
721 
728 
729  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
730  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
731 
732  const TmpType tmp( serial( rhs ) );
733  assign( ~lhs, tmp );
734  }
736  //**********************************************************************************************
737 
738  //**Restructuring assignment to column-major matrices*******************************************
753  template< typename MT > // Type of the target matrix
754  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
755  assign( Matrix<MT,true>& lhs, const SMatDMatMultExpr& rhs )
756  {
758 
760 
761  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
762  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
763 
764  if( IsSymmetric<MT1>::value )
765  assign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
766  else
767  assign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
768  }
770  //**********************************************************************************************
771 
772  //**Addition assignment to dense matrices*******************************************************
785  template< typename MT // Type of the target dense matrix
786  , bool SO > // Storage order of the target dense matrix
787  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
788  {
790 
791  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
792  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
793 
794  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
795  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
796 
797  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
798  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
799  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
800  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
801  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
802  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
803 
804  SMatDMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
805  }
807  //**********************************************************************************************
808 
809  //**Default addition assignment to row-major dense matrices*************************************
823  template< typename MT3 // Type of the left-hand side target matrix
824  , typename MT4 // Type of the left-hand side matrix operand
825  , typename MT5 > // Type of the right-hand side matrix operand
826  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
827  selectAddAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
828  {
829  typedef typename MT4::ConstIterator ConstIterator;
830 
831  for( size_t i=0UL; i<A.rows(); ++i )
832  {
833  const ConstIterator end( A.end(i) );
834 
835  for( size_t j=0UL; j<B.columns(); ++j )
836  {
837  ConstIterator element( A.begin(i) );
838 
839  for( ; element!=end; ++element ) {
840  (~C)(i,j) += element->value() * B(element->index(),j);
841  }
842  }
843  }
844  }
846  //**********************************************************************************************
847 
848  //**Optimized addition assignment to row-major dense matrices***********************************
862  template< typename MT3 // Type of the left-hand side target matrix
863  , typename MT4 // Type of the left-hand side matrix operand
864  , typename MT5 > // Type of the right-hand side matrix operand
865  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
866  selectAddAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
867  {
868  typedef typename MT4::ConstIterator ConstIterator;
869 
870  const size_t jend( B.columns() & size_t(-4) );
871  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jend, "Invalid end calculation" );
872 
873  for( size_t i=0UL; i<A.rows(); ++i )
874  {
875  const ConstIterator end( A.end(i) );
876  ConstIterator element( A.begin(i) );
877 
878  const size_t kend( A.nonZeros(i) & size_t(-4) );
879 
880  for( size_t k=0UL; k<kend; k+=4UL ) {
881  const size_t i1( element->index() );
882  const ET1 v1( element->value() );
883  ++element;
884  const size_t i2( element->index() );
885  const ET1 v2( element->value() );
886  ++element;
887  const size_t i3( element->index() );
888  const ET1 v3( element->value() );
889  ++element;
890  const size_t i4( element->index() );
891  const ET1 v4( element->value() );
892  ++element;
893 
894  for( size_t j=0UL; j<jend; j+=4UL ) {
895  (~C)(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
896  (~C)(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
897  (~C)(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
898  (~C)(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
899  }
900  for( size_t j=jend; j<B.columns(); ++j ) {
901  (~C)(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
902  }
903  }
904 
905  for( ; element!=end; ++element ) {
906  for( size_t j=0UL; j<jend; j+=4UL ) {
907  (~C)(i,j ) += element->value() * B(element->index(),j );
908  (~C)(i,j+1UL) += element->value() * B(element->index(),j+1UL);
909  (~C)(i,j+2UL) += element->value() * B(element->index(),j+2UL);
910  (~C)(i,j+3UL) += element->value() * B(element->index(),j+3UL);
911  }
912  for( size_t j=jend; j<B.columns(); ++j ) {
913  (~C)(i,j) += element->value() * B(element->index(),j);
914  }
915  }
916  }
917  }
919  //**********************************************************************************************
920 
921  //**Vectorized addition assignment to row-major dense matrices**********************************
935  template< typename MT3 // Type of the left-hand side target matrix
936  , typename MT4 // Type of the left-hand side matrix operand
937  , typename MT5 > // Type of the right-hand side matrix operand
938  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
939  selectAddAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
940  {
941  typedef IntrinsicTrait<ElementType> IT;
942  typedef typename MT4::ConstIterator ConstIterator;
943 
944  const size_t N( B.columns() );
945 
946  for( size_t i=0UL; i<A.rows(); ++i )
947  {
948  const ConstIterator end( A.end(i) );
949  ConstIterator element( A.begin(i) );
950 
951  const size_t kend( A.nonZeros(i) & size_t(-4) );
952 
953  for( size_t k=0UL; k<kend; k+=4UL ) {
954  const size_t i1( element->index() );
955  const IntrinsicType v1( set( element->value() ) );
956  ++element;
957  const size_t i2( element->index() );
958  const IntrinsicType v2( set( element->value() ) );
959  ++element;
960  const size_t i3( element->index() );
961  const IntrinsicType v3( set( element->value() ) );
962  ++element;
963  const size_t i4( element->index() );
964  const IntrinsicType v4( set( element->value() ) );
965  ++element;
966 
967  for( size_t j=0UL; j<N; j+=IT::size ) {
968  (~C).store( i, j, (~C).load(i,j) + v1 * B.load(i1,j) + v2 * B.load(i2,j) + v3 * B.load(i3,j) + v4 * B.load(i4,j) );
969  }
970  }
971 
972  for( ; element!=end; ++element ) {
973  const size_t i1( element->index() );
974  const IntrinsicType v1( set( element->value() ) );
975 
976  for( size_t j=0UL; j<N; j+=IT::size ) {
977  (~C).store( i, j, (~C).load(i,j) + v1 * B.load(i1,j) );
978  }
979  }
980  }
981  }
983  //**********************************************************************************************
984 
985  //**Default addition assignment to column-major dense matrices**********************************
999  template< typename MT3 // Type of the left-hand side target matrix
1000  , typename MT4 // Type of the left-hand side matrix operand
1001  , typename MT5 > // Type of the right-hand side matrix operand
1002  static inline void selectAddAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1003  {
1004  typedef typename MT4::ConstIterator ConstIterator;
1005 
1006  const size_t jend( B.columns() & size_t(-4) );
1007  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jend, "Invalid end calculation" );
1008 
1009  for( size_t j=0UL; j<jend; j+=4UL ) {
1010  for( size_t i=0UL; i<A.rows(); ++i )
1011  {
1012  ConstIterator element( A.begin(i) );
1013  const ConstIterator end( A.end(i) );
1014 
1015  for( ; element!=end; ++element ) {
1016  (~C)(i,j ) += element->value() * B(element->index(),j );
1017  (~C)(i,j+1UL) += element->value() * B(element->index(),j+1UL);
1018  (~C)(i,j+2UL) += element->value() * B(element->index(),j+2UL);
1019  (~C)(i,j+3UL) += element->value() * B(element->index(),j+3UL);
1020  }
1021  }
1022  }
1023 
1024  for( size_t j=jend; j<B.columns(); ++j ) {
1025  for( size_t i=0UL; i<A.rows(); ++i )
1026  {
1027  ConstIterator element( A.begin(i) );
1028  const ConstIterator end( A.end(i) );
1029 
1030  for( ; element!=end; ++element )
1031  (~C)(i,j) += element->value() * B(element->index(),j);
1032  }
1033  }
1034  }
1036  //**********************************************************************************************
1037 
1038  //**Restructuring addition assignment to column-major matrices**********************************
1053  template< typename MT > // Type of the target matrix
1054  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1055  addAssign( Matrix<MT,true>& lhs, const SMatDMatMultExpr& rhs )
1056  {
1058 
1060 
1061  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1062  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1063 
1064  if( IsSymmetric<MT1>::value )
1065  addAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1066  else
1067  addAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1068  }
1070  //**********************************************************************************************
1071 
1072  //**Addition assignment to sparse matrices******************************************************
1073  // No special implementation for the addition assignment to sparse matrices.
1074  //**********************************************************************************************
1075 
1076  //**Subtraction assignment to dense matrices****************************************************
1089  template< typename MT // Type of the target dense matrix
1090  , bool SO > // Storage order of the target dense matrix
1091  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1092  {
1094 
1095  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1096  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1097 
1098  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
1099  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
1100 
1101  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1102  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1103  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1104  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1105  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1106  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1107 
1108  SMatDMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1109  }
1111  //**********************************************************************************************
1112 
1113  //**Default subtraction assignment to row-major dense matrices**********************************
1127  template< typename MT3 // Type of the left-hand side target matrix
1128  , typename MT4 // Type of the left-hand side matrix operand
1129  , typename MT5 > // Type of the right-hand side matrix operand
1130  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
1131  selectSubAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
1132  {
1133  typedef typename MT4::ConstIterator ConstIterator;
1134 
1135  for( size_t i=0UL; i<A.rows(); ++i )
1136  {
1137  const ConstIterator end( A.end(i) );
1138 
1139  for( size_t j=0UL; j<B.columns(); ++j )
1140  {
1141  ConstIterator element( A.begin(i) );
1142 
1143  for( ; element!=end; ++element ) {
1144  (~C)(i,j) -= element->value() * B(element->index(),j);
1145  }
1146  }
1147  }
1148  }
1150  //**********************************************************************************************
1151 
1152  //**Optimized subtraction assignment to row-major dense matrices********************************
1166  template< typename MT3 // Type of the left-hand side target matrix
1167  , typename MT4 // Type of the left-hand side matrix operand
1168  , typename MT5 > // Type of the right-hand side matrix operand
1169  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
1170  selectSubAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
1171  {
1172  typedef typename MT4::ConstIterator ConstIterator;
1173 
1174  const size_t jend( B.columns() & size_t(-4) );
1175  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jend, "Invalid end calculation" );
1176 
1177  for( size_t i=0UL; i<A.rows(); ++i )
1178  {
1179  const ConstIterator end( A.end(i) );
1180  ConstIterator element( A.begin(i) );
1181 
1182  const size_t kend( A.nonZeros(i) & size_t(-4) );
1183 
1184  for( size_t k=0UL; k<kend; k+=4UL ) {
1185  const size_t i1( element->index() );
1186  const ET1 v1( element->value() );
1187  ++element;
1188  const size_t i2( element->index() );
1189  const ET1 v2( element->value() );
1190  ++element;
1191  const size_t i3( element->index() );
1192  const ET1 v3( element->value() );
1193  ++element;
1194  const size_t i4( element->index() );
1195  const ET1 v4( element->value() );
1196  ++element;
1197 
1198  for( size_t j=0UL; j<jend; j+=4UL ) {
1199  (~C)(i,j ) -= v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1200  (~C)(i,j+1UL) -= v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1201  (~C)(i,j+2UL) -= v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1202  (~C)(i,j+3UL) -= v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1203  }
1204  for( size_t j=jend; j<B.columns(); ++j ) {
1205  (~C)(i,j) -= v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1206  }
1207  }
1208 
1209  for( ; element!=end; ++element ) {
1210  for( size_t j=0UL; j<jend; j+=4UL ) {
1211  (~C)(i,j ) -= element->value() * B(element->index(),j );
1212  (~C)(i,j+1UL) -= element->value() * B(element->index(),j+1UL);
1213  (~C)(i,j+2UL) -= element->value() * B(element->index(),j+2UL);
1214  (~C)(i,j+3UL) -= element->value() * B(element->index(),j+3UL);
1215  }
1216  for( size_t j=jend; j<B.columns(); ++j ) {
1217  (~C)(i,j) -= element->value() * B(element->index(),j);
1218  }
1219  }
1220  }
1221  }
1223  //**********************************************************************************************
1224 
1225  //**Vectorized subtraction assignment to row-major dense matrices*******************************
1239  template< typename MT3 // Type of the left-hand side target matrix
1240  , typename MT4 // Type of the left-hand side matrix operand
1241  , typename MT5 > // Type of the right-hand side matrix operand
1242  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
1243  selectSubAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
1244  {
1245  typedef IntrinsicTrait<ElementType> IT;
1246  typedef typename MT4::ConstIterator ConstIterator;
1247 
1248  const size_t N( B.columns() );
1249 
1250  for( size_t i=0UL; i<A.rows(); ++i )
1251  {
1252  const ConstIterator end( A.end(i) );
1253  ConstIterator element( A.begin(i) );
1254 
1255  const size_t kend( A.nonZeros(i) & size_t(-4) );
1256 
1257  for( size_t k=0UL; k<kend; k+=4UL ) {
1258  const size_t i1( element->index() );
1259  const IntrinsicType v1( set( element->value() ) );
1260  ++element;
1261  const size_t i2( element->index() );
1262  const IntrinsicType v2( set( element->value() ) );
1263  ++element;
1264  const size_t i3( element->index() );
1265  const IntrinsicType v3( set( element->value() ) );
1266  ++element;
1267  const size_t i4( element->index() );
1268  const IntrinsicType v4( set( element->value() ) );
1269  ++element;
1270 
1271  for( size_t j=0UL; j<N; j+=IT::size ) {
1272  (~C).store( i, j, (~C).load(i,j) - v1 * B.load(i1,j) - v2 * B.load(i2,j) - v3 * B.load(i3,j) - v4 * B.load(i4,j) );
1273  }
1274  }
1275 
1276  for( ; element!=end; ++element ) {
1277  const size_t i1( element->index() );
1278  const IntrinsicType v1( set( element->value() ) );
1279 
1280  for( size_t j=0UL; j<N; j+=IT::size ) {
1281  (~C).store( i, j, (~C).load(i,j) - v1 * B.load(i1,j) );
1282  }
1283  }
1284  }
1285  }
1287  //**********************************************************************************************
1288 
1289  //**Default subtraction assignment to column-major dense matrices*******************************
1303  template< typename MT3 // Type of the left-hand side target matrix
1304  , typename MT4 // Type of the left-hand side matrix operand
1305  , typename MT5 > // Type of the right-hand side matrix operand
1306  static inline void selectSubAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1307  {
1308  typedef typename MT4::ConstIterator ConstIterator;
1309 
1310  const size_t jend( B.columns() & size_t(-4) );
1311  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jend, "Invalid end calculation" );
1312 
1313  for( size_t j=0UL; j<jend; j+=4UL ) {
1314  for( size_t i=0UL; i<A.rows(); ++i )
1315  {
1316  ConstIterator element( A.begin(i) );
1317  const ConstIterator end( A.end(i) );
1318 
1319  for( ; element!=end; ++element ) {
1320  (~C)(i,j ) -= element->value() * B(element->index(),j );
1321  (~C)(i,j+1UL) -= element->value() * B(element->index(),j+1UL);
1322  (~C)(i,j+2UL) -= element->value() * B(element->index(),j+2UL);
1323  (~C)(i,j+3UL) -= element->value() * B(element->index(),j+3UL);
1324  }
1325  }
1326  }
1327 
1328  for( size_t j=jend; j<B.columns(); ++j ) {
1329  for( size_t i=0UL; i<A.rows(); ++i )
1330  {
1331  ConstIterator element( A.begin(i) );
1332  const ConstIterator end( A.end(i) );
1333 
1334  for( ; element!=end; ++element )
1335  (~C)(i,j) -= element->value() * B(element->index(),j);
1336  }
1337  }
1338  }
1340  //**********************************************************************************************
1341 
1342  //**Restructuring subtraction assignment to column-major matrices*******************************
1357  template< typename MT > // Type of the target matrix
1358  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1359  subAssign( Matrix<MT,true>& lhs, const SMatDMatMultExpr& rhs )
1360  {
1362 
1364 
1365  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1366  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1367 
1368  if( IsSymmetric<MT1>::value )
1369  subAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1370  else
1371  subAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1372  }
1374  //**********************************************************************************************
1375 
1376  //**Subtraction assignment to sparse matrices***************************************************
1377  // No special implementation for the subtraction assignment to sparse matrices.
1378  //**********************************************************************************************
1379 
1380  //**Multiplication assignment to dense matrices*************************************************
1381  // No special implementation for the multiplication assignment to dense matrices.
1382  //**********************************************************************************************
1383 
1384  //**Multiplication assignment to sparse matrices************************************************
1385  // No special implementation for the multiplication assignment to sparse matrices.
1386  //**********************************************************************************************
1387 
1388  //**SMP assignment to dense matrices************************************************************
1403  template< typename MT // Type of the target dense matrix
1404  , bool SO > // Storage order of the target dense matrix
1405  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1406  smpAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1407  {
1409 
1410  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1411  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1412 
1413  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1414  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1415 
1416  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1417  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1418  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1419  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1420  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1421  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1422 
1423  smpAssign( ~lhs, A * B );
1424  }
1426  //**********************************************************************************************
1427 
1428  //**SMP assignment to sparse matrices***********************************************************
1443  template< typename MT // Type of the target sparse matrix
1444  , bool SO > // Storage order of the target sparse matrix
1445  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1446  smpAssign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1447  {
1449 
1450  typedef typename SelectType< SO, OppositeType, ResultType >::Type TmpType;
1451 
1458 
1459  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1460  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1461 
1462  const TmpType tmp( rhs );
1463  smpAssign( ~lhs, tmp );
1464  }
1466  //**********************************************************************************************
1467 
1468  //**Restructuring SMP assignment to column-major matrices***************************************
1483  template< typename MT > // Type of the target matrix
1484  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1485  smpAssign( Matrix<MT,true>& lhs, const SMatDMatMultExpr& rhs )
1486  {
1488 
1490 
1491  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1492  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1493 
1494  if( IsSymmetric<MT1>::value )
1495  smpAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1496  else
1497  smpAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1498  }
1500  //**********************************************************************************************
1501 
1502  //**SMP addition assignment to dense matrices***************************************************
1518  template< typename MT // Type of the target dense matrix
1519  , bool SO > // Storage order of the target dense matrix
1520  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1521  smpAddAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1522  {
1524 
1525  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1526  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1527 
1528  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1529  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1530 
1531  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1532  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1533  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1534  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1535  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1536  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1537 
1538  smpAddAssign( ~lhs, A * B );
1539  }
1541  //**********************************************************************************************
1542 
1543  //**Restructuring SMP addition assignment to column-major matrices******************************
1558  template< typename MT > // Type of the target matrix
1559  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1560  smpAddAssign( Matrix<MT,true>& lhs, const SMatDMatMultExpr& rhs )
1561  {
1563 
1565 
1566  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1567  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1568 
1569  if( IsSymmetric<MT1>::value )
1570  smpAddAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1571  else
1572  smpAddAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1573  }
1575  //**********************************************************************************************
1576 
1577  //**SMP addition assignment to sparse matrices**************************************************
1578  // No special implementation for the SMP addition assignment to sparse matrices.
1579  //**********************************************************************************************
1580 
1581  //**SMP subtraction assignment to dense matrices************************************************
1597  template< typename MT // Type of the target dense matrix
1598  , bool SO > // Storage order of the target dense matrix
1599  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1600  smpSubAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1601  {
1603 
1604  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1605  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1606 
1607  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1608  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1609 
1610  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1611  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1612  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1613  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1614  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1615  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1616 
1617  smpSubAssign( ~lhs, A * B );
1618  }
1620  //**********************************************************************************************
1621 
1622  //**Restructuring SMP subtraction assignment to column-major matrices***************************
1637  template< typename MT > // Type of the target matrix
1638  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1639  smpSubAssign( Matrix<MT,true>& lhs, const SMatDMatMultExpr& rhs )
1640  {
1642 
1644 
1645  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1646  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1647 
1648  if( IsSymmetric<MT1>::value )
1649  smpSubAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1650  else
1651  smpSubAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1652  }
1654  //**********************************************************************************************
1655 
1656  //**SMP subtraction assignment to sparse matrices***********************************************
1657  // No special implementation for the SMP subtraction assignment to sparse matrices.
1658  //**********************************************************************************************
1659 
1660  //**SMP multiplication assignment to dense matrices*********************************************
1661  // No special implementation for the SMP multiplication assignment to dense matrices.
1662  //**********************************************************************************************
1663 
1664  //**SMP multiplication assignment to sparse matrices********************************************
1665  // No special implementation for the SMP multiplication assignment to sparse matrices.
1666  //**********************************************************************************************
1667 
1668  //**Compile time checks*************************************************************************
1676  //**********************************************************************************************
1677 };
1678 //*************************************************************************************************
1679 
1680 
1681 
1682 
1683 //=================================================================================================
1684 //
1685 // GLOBAL BINARY ARITHMETIC OPERATORS
1686 //
1687 //=================================================================================================
1688 
1689 //*************************************************************************************************
1718 template< typename T1 // Type of the left-hand side sparse matrix
1719  , typename T2 > // Type of the right-hand side dense matrix
1720 inline const SMatDMatMultExpr<T1,T2>
1722 {
1724 
1725  if( (~lhs).columns() != (~rhs).rows() )
1726  throw std::invalid_argument( "Matrix sizes do not match" );
1727 
1728  return SMatDMatMultExpr<T1,T2>( ~lhs, ~rhs );
1729 }
1730 //*************************************************************************************************
1731 
1732 
1733 
1734 
1735 //=================================================================================================
1736 //
1737 // ROWS SPECIALIZATIONS
1738 //
1739 //=================================================================================================
1740 
1741 //*************************************************************************************************
1743 template< typename MT1, typename MT2 >
1744 struct Rows< SMatDMatMultExpr<MT1,MT2> >
1745  : public Rows<MT1>
1746 {};
1748 //*************************************************************************************************
1749 
1750 
1751 
1752 
1753 //=================================================================================================
1754 //
1755 // COLUMNS SPECIALIZATIONS
1756 //
1757 //=================================================================================================
1758 
1759 //*************************************************************************************************
1761 template< typename MT1, typename MT2 >
1762 struct Columns< SMatDMatMultExpr<MT1,MT2> >
1763  : public Columns<MT2>
1764 {};
1766 //*************************************************************************************************
1767 
1768 
1769 
1770 
1771 //=================================================================================================
1772 //
1773 // ISLOWER SPECIALIZATIONS
1774 //
1775 //=================================================================================================
1776 
1777 //*************************************************************************************************
1779 template< typename MT1, typename MT2 >
1780 struct IsLower< SMatDMatMultExpr<MT1,MT2> >
1781  : public IsTrue< IsLower<MT1>::value && IsLower<MT2>::value >
1782 {};
1784 //*************************************************************************************************
1785 
1786 
1787 
1788 
1789 //=================================================================================================
1790 //
1791 // ISUPPER SPECIALIZATIONS
1792 //
1793 //=================================================================================================
1794 
1795 //*************************************************************************************************
1797 template< typename MT1, typename MT2 >
1798 struct IsUpper< SMatDMatMultExpr<MT1,MT2> >
1799  : public IsTrue< IsUpper<MT1>::value && IsUpper<MT2>::value >
1800 {};
1802 //*************************************************************************************************
1803 
1804 
1805 
1806 
1807 //=================================================================================================
1808 //
1809 // EXPRESSION TRAIT SPECIALIZATIONS
1810 //
1811 //=================================================================================================
1812 
1813 //*************************************************************************************************
1815 template< typename MT1, typename MT2, typename VT >
1816 struct DMatDVecMultExprTrait< SMatDMatMultExpr<MT1,MT2>, VT >
1817 {
1818  public:
1819  //**********************************************************************************************
1820  typedef typename SelectType< IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
1821  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value &&
1822  IsDenseVector<VT>::value && IsColumnVector<VT>::value
1823  , typename SMatDVecMultExprTrait< MT1, typename DMatDVecMultExprTrait<MT2,VT>::Type >::Type
1824  , INVALID_TYPE >::Type Type;
1825  //**********************************************************************************************
1826 };
1828 //*************************************************************************************************
1829 
1830 
1831 //*************************************************************************************************
1833 template< typename MT1, typename MT2, typename VT >
1834 struct DMatSVecMultExprTrait< SMatDMatMultExpr<MT1,MT2>, VT >
1835 {
1836  public:
1837  //**********************************************************************************************
1838  typedef typename SelectType< IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
1839  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value &&
1840  IsSparseVector<VT>::value && IsColumnVector<VT>::value
1841  , typename SMatDVecMultExprTrait< MT1, typename DMatSVecMultExprTrait<MT2,VT>::Type >::Type
1842  , INVALID_TYPE >::Type Type;
1843  //**********************************************************************************************
1844 };
1846 //*************************************************************************************************
1847 
1848 
1849 //*************************************************************************************************
1851 template< typename VT, typename MT1, typename MT2 >
1852 struct TDVecDMatMultExprTrait< VT, SMatDMatMultExpr<MT1,MT2> >
1853 {
1854  public:
1855  //**********************************************************************************************
1856  typedef typename SelectType< IsDenseVector<VT>::value && IsRowVector<VT>::value &&
1857  IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
1858  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value
1859  , typename TDVecDMatMultExprTrait< typename TDVecSMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
1860  , INVALID_TYPE >::Type Type;
1861  //**********************************************************************************************
1862 };
1864 //*************************************************************************************************
1865 
1866 
1867 //*************************************************************************************************
1869 template< typename VT, typename MT1, typename MT2 >
1870 struct TSVecDMatMultExprTrait< VT, SMatDMatMultExpr<MT1,MT2> >
1871 {
1872  public:
1873  //**********************************************************************************************
1874  typedef typename SelectType< IsSparseVector<VT>::value && IsRowVector<VT>::value &&
1875  IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
1876  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value
1877  , typename TSVecDMatMultExprTrait< typename TSVecSMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
1878  , INVALID_TYPE >::Type Type;
1879  //**********************************************************************************************
1880 };
1882 //*************************************************************************************************
1883 
1884 
1885 //*************************************************************************************************
1887 template< typename MT1, typename MT2, bool AF >
1888 struct SubmatrixExprTrait< SMatDMatMultExpr<MT1,MT2>, AF >
1889 {
1890  public:
1891  //**********************************************************************************************
1892  typedef typename MultExprTrait< typename SubmatrixExprTrait<const MT1,AF>::Type
1893  , typename SubmatrixExprTrait<const MT2,AF>::Type >::Type Type;
1894  //**********************************************************************************************
1895 };
1897 //*************************************************************************************************
1898 
1899 
1900 //*************************************************************************************************
1902 template< typename MT1, typename MT2 >
1903 struct RowExprTrait< SMatDMatMultExpr<MT1,MT2> >
1904 {
1905  public:
1906  //**********************************************************************************************
1907  typedef typename MultExprTrait< typename RowExprTrait<const MT1>::Type, MT2 >::Type Type;
1908  //**********************************************************************************************
1909 };
1911 //*************************************************************************************************
1912 
1913 
1914 //*************************************************************************************************
1916 template< typename MT1, typename MT2 >
1917 struct ColumnExprTrait< SMatDMatMultExpr<MT1,MT2> >
1918 {
1919  public:
1920  //**********************************************************************************************
1921  typedef typename MultExprTrait< MT1, typename ColumnExprTrait<const MT2>::Type >::Type Type;
1922  //**********************************************************************************************
1923 };
1925 //*************************************************************************************************
1926 
1927 } // namespace blaze
1928 
1929 #endif
SelectType< IsExpression< MT1 >::value, const MT1, const MT1 & >::Type LeftOperand
Composite type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:225
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:89
Header file for the SMatDVecMultExprTrait class template.
Header file for the Rows type trait.
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:4838
BLAZE_ALWAYS_INLINE size_t size(const Vector< VT, TF > &vector)
Returns the current size/dimension of the vector.
Definition: Vector.h:258
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:258
LeftOperand leftOperand() const
Returns the left-hand side sparse matrix operand.
Definition: SMatDMatMultExpr.h:338
Header file for the IsSparseMatrix type trait.
Efficient implementation of a compressed matrix.The CompressedMatrix class template is the represent...
Definition: CompressedMatrix.h:205
bool canSMPAssign() const
Returns whether the expression can be used in SMP assignments.
Definition: SMatDMatMultExpr.h:392
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:79
Header file for the ColumnExprTrait class template.
Header file for the IsSame and IsStrictlySame type traits.
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:2478
Header file for the IsRowVector type trait.
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:257
Header file for the TDVecSMatMultExprTrait class template.
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:695
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Expression object for sparse matrix-dense matrix multiplications.The SMatDMatMultExpr class represent...
Definition: Forward.h:89
Type relationship analysis.This class tests if the two data types A and B are equal. For this type comparison, the cv-qualifiers of both data types are ignored. If A and B are the same data type (ignoring the cv-qualifiers), then the value member enumeration is set to 1, the nested type definition Type is TrueType, and the class derives from TrueType. Otherwise value is set to 0, Type is FalseType, and the class derives from FalseType.
Definition: IsSame.h:158
Header file for the RequiresEvaluation type trait.
Header file for the TSVecSMatMultExprTrait class template.
const ElementType ReturnType
Return type for expression template evaluations.
Definition: SMatDMatMultExpr.h:221
SelectType< IsExpression< MT2 >::value, const MT2, const MT2 & >::Type RightOperand
Composite type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:228
const ResultType CompositeType
Data type for composite expression templates.
Definition: SMatDMatMultExpr.h:222
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:107
Constraint on the data type.
Constraint on the data type.
Header file for the MultExprTrait class template.
bool isAliased(const T *alias) const
Returns whether the expression is aliased with the given address alias.
Definition: SMatDMatMultExpr.h:372
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:90
RT1::ElementType ET1
Element type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:121
Compile time type selection.The SelectType class template selects one of the two given types T1 and T...
Definition: SelectType.h:59
Header file for the DisableIf class template.
Header file for the multiplication trait.
Header file for the IsSymmetric type trait.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: StorageOrder.h:161
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2482
Header file for the DenseMatrix base class.
MT2::ResultType RT2
Result type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:120
BLAZE_ALWAYS_INLINE void assign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the assignment of a matrix to a matrix.
Definition: Matrix.h:635
Header file for the Columns type trait.
Header file for the DMatDVecMultExprTrait class template.
Header file for the IsLower type trait.
RightOperand rightOperand() const
Returns the right-hand side dense matrix operand.
Definition: SMatDMatMultExpr.h:348
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:78
Constraints on the storage order of matrix types.
MultTrait< RT1, RT2 >::Type ResultType
Result type for expression template evaluations.
Definition: SMatDMatMultExpr.h:216
SelectType< evaluateLeft, const RT1, CT1 >::Type LT
Type for the assignment of the left-hand side sparse matrix operand.
Definition: SMatDMatMultExpr.h:231
RightOperand rhs_
Right-hand side dense matrix of the multiplication expression.
Definition: SMatDMatMultExpr.h:400
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:2476
bool isAligned() const
Returns whether the operands of the expression are properly aligned in memory.
Definition: SMatDMatMultExpr.h:382
Header file for the SelectType class template.
Header file for the RowExprTrait class template.
Header file for all forward declarations for expression class templates.
IntrinsicTrait< ElementType >::Type IntrinsicType
Resulting intrinsic element type.
Definition: SMatDMatMultExpr.h:220
Header file for the IsDenseMatrix type trait.
Header file for the EnableIf class template.
const size_t SMP_SMATDMATMULT_THRESHOLD
SMP row-major sparse matrix/row-major dense matrix multiplication threshold.This threshold specifies ...
Definition: Thresholds.h:1018
Header file for the serial shim.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:165
ResultType::ElementType ElementType
Resulting element type.
Definition: SMatDMatMultExpr.h:219
BLAZE_ALWAYS_INLINE EnableIf< IsIntegral< T >, Set< T, sizeof(T)> >::Type::Type set(T value)
Sets all values in the vector to the given integral value.
Definition: Set.h:211
ResultType::OppositeType OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: SMatDMatMultExpr.h:217
EnableIf< IsDenseMatrix< MT1 > >::Type smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
Header file for the IsSparseVector type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type, a compilation error is created.
Definition: Symmetric.h:116
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: StorageOrder.h:81
LeftOperand lhs_
Left-hand side sparse matrix of the multiplication expression.
Definition: SMatDMatMultExpr.h:399
Removal of reference modifiers.The RemoveCV type trait removes any reference modifiers from the given...
Definition: RemoveReference.h:69
Intrinsic characteristics of data types.The IntrinsicTrait class template provides the intrinsic char...
Definition: IntrinsicTrait.h:749
bool canAlias(const T *alias) const
Returns whether the expression can alias with the given address alias.
Definition: SMatDMatMultExpr.h:360
Header file for run time assertion macros.
EnableIf< IsDenseMatrix< MT1 > >::Type smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Utility type for generic codes.
Base template for the MultTrait class.
Definition: MultTrait.h:142
SelectType< evaluateRight, const RT2, CT2 >::Type RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: SMatDMatMultExpr.h:234
BLAZE_ALWAYS_INLINE void addAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the addition assignment of a matrix to a matrix.
Definition: Matrix.h:742
Substitution Failure Is Not An Error (SFINAE) class.The EnableIf class template is an auxiliary tool ...
Definition: EnableIf.h:184
Header file for the reset shim.
size_t rows() const
Returns the current number of rows of the matrix.
Definition: SMatDMatMultExpr.h:318
MT1::ResultType RT1
Result type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:119
BLAZE_ALWAYS_INLINE void reset(const NonNumericProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: NonNumericProxy.h:833
Header file for the RemoveReference type trait.
Substitution Failure Is Not An Error (SFINAE) class.The DisableIf class template is an auxiliary tool...
Definition: DisableIf.h:184
MT2::CompositeType CT2
Composite type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:124
SMatDMatMultExpr< MT1, MT2 > This
Type of this SMatDMatMultExpr instance.
Definition: SMatDMatMultExpr.h:215
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:283
Header file for the IsDenseVector type trait.
Header file for all intrinsic functionality.
RT2::ElementType ET2
Element type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:122
Header file for the IsRowMajorMatrix type trait.
const DMatTransExpr< MT,!SO > trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:932
Header file for the IsComputation type trait class.
SMatDMatMultExpr(const MT1 &lhs, const MT2 &rhs)
Constructor for the SMatDMatMultExpr class.
Definition: SMatDMatMultExpr.h:255
Header file for the TDVecDMatMultExprTrait class template.
EnableIf< IsDenseMatrix< MT1 > >::Type smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
This ResultType
Result type for expression template evaluations.
Definition: CompressedMatrix.h:2473
Header file for the IsTrue value trait.
MT1::CompositeType CT1
Composite type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:123
Header file for basic type definitions.
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: SMatDMatMultExpr.h:270
Header file for the TSVecDMatMultExprTrait class template.
ResultType::TransposeType TransposeType
Transpose type for expression template evaluations.
Definition: SMatDMatMultExpr.h:218
Header file for the IsUpper type trait.
Header file for the DMatSVecMultExprTrait class template.
Header file for the IsColumnVector type trait.
size_t columns() const
Returns the current number of columns of the matrix.
Definition: SMatDMatMultExpr.h:328
Constraint on the data type.
BLAZE_ALWAYS_INLINE EnableIf< IsIntegral< T > >::Type store(T *address, const typename Store< T, sizeof(T)>::Type &value)
Aligned store of a vector of integral values.
Definition: Store.h:225
Header file for the IsResizable type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:79
Header file for the IsExpression type trait class.
Header file for the FunctionTrace class.
BLAZE_ALWAYS_INLINE void subAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the subtraction assignment of a matrix to matrix.
Definition: Matrix.h:849