All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
SMatDMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
20 //=================================================================================================
21 
22 #ifndef _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
23 #define _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
24 
25 
26 //*************************************************************************************************
27 // Includes
28 //*************************************************************************************************
29 
30 #include <stdexcept>
38 #include <blaze/math/Intrinsics.h>
39 #include <blaze/math/shims/Reset.h>
61 #include <blaze/util/Assert.h>
62 #include <blaze/util/EnableIf.h>
63 #include <blaze/util/InvalidType.h>
65 #include <blaze/util/SelectType.h>
66 #include <blaze/util/Types.h>
69 
70 
71 namespace blaze {
72 
73 //=================================================================================================
74 //
75 // CLASS SMATDMATMULTEXPR
76 //
77 //=================================================================================================
78 
79 //*************************************************************************************************
86 template< typename MT1 // Type of the left-hand side sparse matrix
87  , typename MT2 > // Type of the right-hand side dense matrix
88 class SMatDMatMultExpr : public DenseMatrix< SMatDMatMultExpr<MT1,MT2>, false >
89  , private MatMatMultExpr
90  , private Computation
91 {
92  private:
93  //**Type definitions****************************************************************************
94  typedef typename MT1::ResultType RT1;
95  typedef typename MT2::ResultType RT2;
96  typedef typename MT1::ElementType ET1;
97  typedef typename MT2::ElementType ET2;
98  typedef typename MT1::CompositeType CT1;
99  typedef typename MT2::CompositeType CT2;
100  //**********************************************************************************************
101 
102  //**********************************************************************************************
104 
105 
107  template< typename T1, typename T2, typename T3 >
108  struct UseVectorizedKernel {
109  enum { value = T1::vectorizable && T3::vectorizable &&
116  };
118  //**********************************************************************************************
119 
120  //**********************************************************************************************
122 
123 
126  template< typename T1, typename T2, typename T3 >
127  struct UseOptimizedKernel {
128  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
131  };
133  //**********************************************************************************************
134 
135  //**********************************************************************************************
137 
138 
140  template< typename T1, typename T2, typename T3 >
141  struct UseDefaultKernel {
142  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
143  !UseOptimizedKernel<T1,T2,T3>::value };
144  };
146  //**********************************************************************************************
147 
148  public:
149  //**Type definitions****************************************************************************
152  typedef typename ResultType::OppositeType OppositeType;
153  typedef typename ResultType::TransposeType TransposeType;
154  typedef typename ResultType::ElementType ElementType;
156  typedef const ElementType ReturnType;
157  typedef const ResultType CompositeType;
158 
160  typedef typename SelectType< IsExpression<MT1>::value, const MT1, const MT1& >::Type LeftOperand;
161 
163  typedef typename SelectType< IsExpression<MT2>::value, const MT2, const MT2& >::Type RightOperand;
164 
166  typedef typename SelectType< IsComputation<MT1>::value, const RT1, CT1 >::Type LT;
167 
169  typedef typename SelectType< IsComputation<MT2>::value, const RT2, CT2 >::Type RT;
170  //**********************************************************************************************
171 
172  //**Compilation flags***************************************************************************
174  enum { vectorizable = 0 };
175  //**********************************************************************************************
176 
177  //**Constructor*********************************************************************************
183  explicit inline SMatDMatMultExpr( const MT1& lhs, const MT2& rhs )
184  : lhs_( lhs ) // Left-hand side sparse matrix of the multiplication expression
185  , rhs_( rhs ) // Right-hand side dense matrix of the multiplication expression
186  {
187  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
188  }
189  //**********************************************************************************************
190 
191  //**Access operator*****************************************************************************
198  inline ReturnType operator()( size_t i, size_t j ) const {
199  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
200  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
201 
202  typedef typename RemoveReference<CT1>::Type::ConstIterator ConstIterator;
203 
204  ElementType tmp = ElementType();
205 
206  // Early exit
207  if( lhs_.columns() == 0 )
208  return tmp;
209 
210  // Fast computation in case the left-hand side sparse matrix directly provides iterators
212  {
213  CT1 A( lhs_ ); // Evaluation of the left-hand side sparse matrix operand
214 
215  const ConstIterator end( A.end(i) );
216  ConstIterator element( A.begin(i) );
217 
218  // Early exit in case row i is empty
219  if( element == end )
220  return tmp;
221 
222  // Calculating element (i,j)
223  tmp = element->value() * rhs_(element->index(),j);
224  ++element;
225  for( ; element!=end; ++element )
226  tmp += element->value() * rhs_(element->index(),j);
227  }
228 
229  // Default computation in case the left-hand side sparse matrix doesn't provide iterators
230  else {
231  tmp = lhs_(i,0) * rhs_(0,j);
232  for( size_t k=1; k<lhs_.columns(); ++k ) {
233  tmp += lhs_(i,k) * rhs_(k,j);
234  }
235  }
236 
237  return tmp;
238  }
239  //**********************************************************************************************
240 
241  //**Rows function*******************************************************************************
246  inline size_t rows() const {
247  return lhs_.rows();
248  }
249  //**********************************************************************************************
250 
251  //**Columns function****************************************************************************
256  inline size_t columns() const {
257  return rhs_.columns();
258  }
259  //**********************************************************************************************
260 
261  //**Left operand access*************************************************************************
266  inline LeftOperand leftOperand() const {
267  return lhs_;
268  }
269  //**********************************************************************************************
270 
271  //**Right operand access************************************************************************
276  inline RightOperand rightOperand() const {
277  return rhs_;
278  }
279  //**********************************************************************************************
280 
281  //**********************************************************************************************
287  template< typename T >
288  inline bool canAlias( const T* alias ) const {
289  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
290  }
291  //**********************************************************************************************
292 
293  //**********************************************************************************************
299  template< typename T >
300  inline bool isAliased( const T* alias ) const {
301  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
302  }
303  //**********************************************************************************************
304 
305  private:
306  //**Member variables****************************************************************************
309  //**********************************************************************************************
310 
311  //**Assignment to dense matrices****************************************************************
323  template< typename MT // Type of the target dense matrix
324  , bool SO > // Storage order of the target dense matrix
325  friend inline void assign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
326  {
328 
329  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
330  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
331 
332  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
333  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
334 
335  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
336  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
337  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
338  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
339  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
340  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
341 
342  SMatDMatMultExpr::selectAssignKernel( ~lhs, A, B );
343  }
345  //**********************************************************************************************
346 
347  //**Default assignment to row-major dense matrices**********************************************
361  template< typename MT3 // Type of the left-hand side target matrix
362  , typename MT4 // Type of the left-hand side matrix operand
363  , typename MT5 > // Type of the right-hand side matrix operand
364  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
365  selectAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
366  {
367  typedef typename RemoveReference<LT>::Type::ConstIterator ConstIterator;
368 
369  for( size_t i=0UL; i<A.rows(); ++i )
370  {
371  const ConstIterator end( A.end(i) );
372 
373  for( size_t j=0UL; j<B.columns(); ++j )
374  {
375  ConstIterator element( A.begin(i) );
376 
377  if( element != end ) {
378  (~C)(i,j) = element->value() * B(element->index(),j);
379  ++element;
380  for( ; element!=end; ++element ) {
381  (~C)(i,j) += element->value() * B(element->index(),j);
382  }
383  }
384  else {
385  reset( (~C)(i,j) );
386  }
387  }
388  }
389  }
391  //**********************************************************************************************
392 
393  //**Optimized assignment to row-major dense matrices********************************************
407  template< typename MT3 // Type of the left-hand side target matrix
408  , typename MT4 // Type of the left-hand side matrix operand
409  , typename MT5 > // Type of the right-hand side matrix operand
410  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
411  selectAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
412  {
413  typedef typename RemoveReference<LT>::Type::ConstIterator ConstIterator;
414 
415  const size_t jend( B.columns() & size_t(-4) );
416  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jend, "Invalid end calculation" );
417 
418  reset( ~C );
419 
420  for( size_t i=0UL; i<A.rows(); ++i )
421  {
422  const ConstIterator end( A.end(i) );
423  ConstIterator element( A.begin(i) );
424 
425  const size_t kend( A.nonZeros(i) & size_t(-4) );
426 
427  for( size_t k=0UL; k<kend; k+=4UL ) {
428  const size_t i1( element->index() );
429  const ET1 v1( element->value() );
430  ++element;
431  const size_t i2( element->index() );
432  const ET1 v2( element->value() );
433  ++element;
434  const size_t i3( element->index() );
435  const ET1 v3( element->value() );
436  ++element;
437  const size_t i4( element->index() );
438  const ET1 v4( element->value() );
439  ++element;
440 
441  for( size_t j=0UL; j<jend; j+=4UL ) {
442  (~C)(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
443  (~C)(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
444  (~C)(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
445  (~C)(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
446  }
447  for( size_t j=jend; j<B.columns(); ++j ) {
448  (~C)(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
449  }
450  }
451 
452  for( ; element!=end; ++element ) {
453  for( size_t j=0UL; j<jend; j+=4UL ) {
454  (~C)(i,j ) += element->value() * B(element->index(),j );
455  (~C)(i,j+1UL) += element->value() * B(element->index(),j+1UL);
456  (~C)(i,j+2UL) += element->value() * B(element->index(),j+2UL);
457  (~C)(i,j+3UL) += element->value() * B(element->index(),j+3UL);
458  }
459  for( size_t j=jend; j<B.columns(); ++j ) {
460  (~C)(i,j) += element->value() * B(element->index(),j);
461  }
462  }
463  }
464  }
466  //**********************************************************************************************
467 
468  //**Vectorized assignment to row-major dense matrices*******************************************
482  template< typename MT3 // Type of the left-hand side target matrix
483  , typename MT4 // Type of the left-hand side matrix operand
484  , typename MT5 > // Type of the right-hand side matrix operand
485  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
486  selectAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
487  {
488  typedef IntrinsicTrait<ElementType> IT;
489  typedef typename RemoveReference<LT>::Type::ConstIterator ConstIterator;
490 
491  const size_t N( B.columns() );
492 
493  reset( ~C );
494 
495  for( size_t i=0UL; i<A.rows(); ++i )
496  {
497  const ConstIterator end( A.end(i) );
498  ConstIterator element( A.begin(i) );
499 
500  const size_t kend( A.nonZeros(i) & size_t(-4) );
501 
502  for( size_t k=0UL; k<kend; k+=4UL ) {
503  const size_t i1( element->index() );
504  const IntrinsicType v1( set( element->value() ) );
505  ++element;
506  const size_t i2( element->index() );
507  const IntrinsicType v2( set( element->value() ) );
508  ++element;
509  const size_t i3( element->index() );
510  const IntrinsicType v3( set( element->value() ) );
511  ++element;
512  const size_t i4( element->index() );
513  const IntrinsicType v4( set( element->value() ) );
514  ++element;
515 
516  for( size_t j=0UL; j<N; j+=IT::size ) {
517  store( &(~C)(i,j), load( &(~C)(i,j) ) + v1 * B.get(i1,j) + v2 * B.get(i2,j) + v3 * B.get(i3,j) + v4 * B.get(i4,j) );
518  }
519  }
520 
521  for( ; element!=end; ++element ) {
522  const size_t i1( element->index() );
523  const IntrinsicType v1( set( element->value() ) );
524 
525  for( size_t j=0UL; j<N; j+=IT::size ) {
526  store( &(~C)(i,j), load( &(~C)(i,j) ) + v1 * B.get(i1,j) );
527  }
528  }
529  }
530  }
532  //**********************************************************************************************
533 
534  //**Default assignment to column-major dense matrices*******************************************
548  template< typename MT3 // Type of the left-hand side target matrix
549  , typename MT4 // Type of the left-hand side matrix operand
550  , typename MT5 > // Type of the right-hand side matrix operand
551  static inline void
552  selectAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
553  {
554  typedef typename RemoveReference<LT>::Type::ConstIterator ConstIterator;
555 
556  const size_t jend( B.columns() & size_t(-4) );
557  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jend, "Invalid end calculation" );
558 
559  for( size_t j=0UL; j<jend; j+=4UL ) {
560  for( size_t i=0UL; i<A.rows(); ++i )
561  {
562  ConstIterator element( A.begin(i) );
563  const ConstIterator end( A.end(i) );
564 
565  if( element == end ) {
566  reset( (~C)(i,j ) );
567  reset( (~C)(i,j+1UL) );
568  reset( (~C)(i,j+2UL) );
569  reset( (~C)(i,j+3UL) );
570  continue;
571  }
572 
573  (~C)(i,j ) = element->value() * B(element->index(),j );
574  (~C)(i,j+1UL) = element->value() * B(element->index(),j+1UL);
575  (~C)(i,j+2UL) = element->value() * B(element->index(),j+2UL);
576  (~C)(i,j+3UL) = element->value() * B(element->index(),j+3UL);
577  ++element;
578  for( ; element!=end; ++element ) {
579  (~C)(i,j ) += element->value() * B(element->index(),j );
580  (~C)(i,j+1UL) += element->value() * B(element->index(),j+1UL);
581  (~C)(i,j+2UL) += element->value() * B(element->index(),j+2UL);
582  (~C)(i,j+3UL) += element->value() * B(element->index(),j+3UL);
583  }
584  }
585  }
586 
587  for( size_t j=jend; j<B.columns(); ++j ) {
588  for( size_t i=0UL; i<A.rows(); ++i )
589  {
590  ConstIterator element( A.begin(i) );
591  const ConstIterator end( A.end(i) );
592 
593  if( element == end ) {
594  reset( (~C)(i,j) );
595  continue;
596  }
597 
598  (~C)(i,j) = element->value() * B(element->index(),j);
599  ++element;
600  for( ; element!=end; ++element )
601  (~C)(i,j) += element->value() * B(element->index(),j);
602  }
603  }
604  }
606  //**********************************************************************************************
607 
608  //**Assignment to sparse matrices***************************************************************
620  template< typename MT // Type of the target sparse matrix
621  , bool SO > // Storage order of the target sparse matrix
622  friend inline void assign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
623  {
625 
626  typedef typename SelectType< SO, OppositeType, ResultType >::Type TmpType;
627 
633  BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE( typename TmpType::CompositeType );
634 
635  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
636  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
637 
638  const TmpType tmp( rhs );
639  assign( ~lhs, tmp );
640  }
642  //**********************************************************************************************
643 
644  //**Addition assignment to dense matrices*******************************************************
656  template< typename MT // Type of the target dense matrix
657  , bool SO > // Storage order of the target dense matrix
658  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
659  {
661 
662  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
663  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
664 
665  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
666  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
667 
668  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
669  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
670  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
671  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
672  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
673  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
674 
675  SMatDMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
676  }
678  //**********************************************************************************************
679 
680  //**Default addition assignment to dense matrices***********************************************
694  template< typename MT3 // Type of the left-hand side target matrix
695  , typename MT4 // Type of the left-hand side matrix operand
696  , typename MT5 > // Type of the right-hand side matrix operand
697  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
698  selectAddAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
699  {
700  typedef typename RemoveReference<LT>::Type::ConstIterator ConstIterator;
701 
702  for( size_t i=0UL; i<A.rows(); ++i )
703  {
704  const ConstIterator end( A.end(i) );
705 
706  for( size_t j=0UL; j<B.columns(); ++j )
707  {
708  ConstIterator element( A.begin(i) );
709 
710  for( ; element!=end; ++element ) {
711  (~C)(i,j) += element->value() * B(element->index(),j);
712  }
713  }
714  }
715  }
717  //**********************************************************************************************
718 
719  //**Optimized addition assignment to dense matrices*********************************************
733  template< typename MT3 // Type of the left-hand side target matrix
734  , typename MT4 // Type of the left-hand side matrix operand
735  , typename MT5 > // Type of the right-hand side matrix operand
736  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
737  selectAddAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
738  {
739  typedef typename RemoveReference<LT>::Type::ConstIterator ConstIterator;
740 
741  const size_t jend( B.columns() & size_t(-4) );
742  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jend, "Invalid end calculation" );
743 
744  for( size_t i=0UL; i<A.rows(); ++i )
745  {
746  const ConstIterator end( A.end(i) );
747  ConstIterator element( A.begin(i) );
748 
749  const size_t kend( A.nonZeros(i) & size_t(-4) );
750 
751  for( size_t k=0UL; k<kend; k+=4UL ) {
752  const size_t i1( element->index() );
753  const ET1 v1( element->value() );
754  ++element;
755  const size_t i2( element->index() );
756  const ET1 v2( element->value() );
757  ++element;
758  const size_t i3( element->index() );
759  const ET1 v3( element->value() );
760  ++element;
761  const size_t i4( element->index() );
762  const ET1 v4( element->value() );
763  ++element;
764 
765  for( size_t j=0UL; j<jend; j+=4UL ) {
766  (~C)(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
767  (~C)(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
768  (~C)(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
769  (~C)(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
770  }
771  for( size_t j=jend; j<B.columns(); ++j ) {
772  (~C)(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
773  }
774  }
775 
776  for( ; element!=end; ++element ) {
777  for( size_t j=0UL; j<jend; j+=4UL ) {
778  (~C)(i,j ) += element->value() * B(element->index(),j );
779  (~C)(i,j+1UL) += element->value() * B(element->index(),j+1UL);
780  (~C)(i,j+2UL) += element->value() * B(element->index(),j+2UL);
781  (~C)(i,j+3UL) += element->value() * B(element->index(),j+3UL);
782  }
783  for( size_t j=jend; j<B.columns(); ++j ) {
784  (~C)(i,j) += element->value() * B(element->index(),j);
785  }
786  }
787  }
788  }
790  //**********************************************************************************************
791 
792  //**Vectorized addition assignment to dense matrices********************************************
806  template< typename MT3 // Type of the left-hand side target matrix
807  , typename MT4 // Type of the left-hand side matrix operand
808  , typename MT5 > // Type of the right-hand side matrix operand
809  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
810  selectAddAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
811  {
812  typedef IntrinsicTrait<ElementType> IT;
813  typedef typename RemoveReference<LT>::Type::ConstIterator ConstIterator;
814 
815  const size_t N( B.columns() );
816 
817  for( size_t i=0UL; i<A.rows(); ++i )
818  {
819  const ConstIterator end( A.end(i) );
820  ConstIterator element( A.begin(i) );
821 
822  const size_t kend( A.nonZeros(i) & size_t(-4) );
823 
824  for( size_t k=0UL; k<kend; k+=4UL ) {
825  const size_t i1( element->index() );
826  const IntrinsicType v1( set( element->value() ) );
827  ++element;
828  const size_t i2( element->index() );
829  const IntrinsicType v2( set( element->value() ) );
830  ++element;
831  const size_t i3( element->index() );
832  const IntrinsicType v3( set( element->value() ) );
833  ++element;
834  const size_t i4( element->index() );
835  const IntrinsicType v4( set( element->value() ) );
836  ++element;
837 
838  for( size_t j=0UL; j<N; j+=IT::size ) {
839  store( &(~C)(i,j), load( &(~C)(i,j) ) + v1 * B.get(i1,j) + v2 * B.get(i2,j) + v3 * B.get(i3,j) + v4 * B.get(i4,j) );
840  }
841  }
842 
843  for( ; element!=end; ++element ) {
844  const size_t i1( element->index() );
845  const IntrinsicType v1( set( element->value() ) );
846 
847  for( size_t j=0UL; j<N; j+=IT::size ) {
848  store( &(~C)(i,j), load( &(~C)(i,j) ) + v1 * B.get(i1,j) );
849  }
850  }
851  }
852  }
854  //**********************************************************************************************
855 
856  //**Default addition assignment to column-major dense matrices**********************************
870  template< typename MT3 // Type of the left-hand side target matrix
871  , typename MT4 // Type of the left-hand side matrix operand
872  , typename MT5 > // Type of the right-hand side matrix operand
873  static inline void
874  selectAddAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
875  {
876  typedef typename RemoveReference<LT>::Type::ConstIterator ConstIterator;
877 
878  const size_t jend( B.columns() & size_t(-4) );
879  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jend, "Invalid end calculation" );
880 
881  for( size_t j=0UL; j<jend; j+=4UL ) {
882  for( size_t i=0UL; i<A.rows(); ++i )
883  {
884  ConstIterator element( A.begin(i) );
885  const ConstIterator end( A.end(i) );
886 
887  for( ; element!=end; ++element ) {
888  (~C)(i,j ) += element->value() * B(element->index(),j );
889  (~C)(i,j+1UL) += element->value() * B(element->index(),j+1UL);
890  (~C)(i,j+2UL) += element->value() * B(element->index(),j+2UL);
891  (~C)(i,j+3UL) += element->value() * B(element->index(),j+3UL);
892  }
893  }
894  }
895 
896  for( size_t j=jend; j<B.columns(); ++j ) {
897  for( size_t i=0UL; i<A.rows(); ++i )
898  {
899  ConstIterator element( A.begin(i) );
900  const ConstIterator end( A.end(i) );
901 
902  for( ; element!=end; ++element )
903  (~C)(i,j) += element->value() * B(element->index(),j);
904  }
905  }
906  }
908  //**********************************************************************************************
909 
910  //**Addition assignment to sparse matrices******************************************************
911  // No special implementation for the addition assignment to sparse matrices.
912  //**********************************************************************************************
913 
914  //**Subtraction assignment to dense matrices****************************************************
926  template< typename MT // Type of the target dense matrix
927  , bool SO > // Storage order of the target dense matrix
928  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
929  {
931 
932  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
933  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
934 
935  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
936  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
937 
938  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
939  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
940  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
941  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
942  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
943  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
944 
945  SMatDMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
946  }
948  //**********************************************************************************************
949 
950  //**Default subtraction assignment to dense matrices********************************************
964  template< typename MT3 // Type of the left-hand side target matrix
965  , typename MT4 // Type of the left-hand side matrix operand
966  , typename MT5 > // Type of the right-hand side matrix operand
967  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
968  selectSubAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
969  {
970  typedef typename RemoveReference<LT>::Type::ConstIterator ConstIterator;
971 
972  for( size_t i=0UL; i<A.rows(); ++i )
973  {
974  const ConstIterator end( A.end(i) );
975 
976  for( size_t j=0UL; j<B.columns(); ++j )
977  {
978  ConstIterator element( A.begin(i) );
979 
980  for( ; element!=end; ++element ) {
981  (~C)(i,j) -= element->value() * B(element->index(),j);
982  }
983  }
984  }
985  }
987  //**********************************************************************************************
988 
989  //**Optimized subtraction assignment to dense matrices******************************************
1003  template< typename MT3 // Type of the left-hand side target matrix
1004  , typename MT4 // Type of the left-hand side matrix operand
1005  , typename MT5 > // Type of the right-hand side matrix operand
1006  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
1007  selectSubAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
1008  {
1009  typedef typename RemoveReference<LT>::Type::ConstIterator ConstIterator;
1010 
1011  const size_t jend( B.columns() & size_t(-4) );
1012  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jend, "Invalid end calculation" );
1013 
1014  for( size_t i=0UL; i<A.rows(); ++i )
1015  {
1016  const ConstIterator end( A.end(i) );
1017  ConstIterator element( A.begin(i) );
1018 
1019  const size_t kend( A.nonZeros(i) & size_t(-4) );
1020 
1021  for( size_t k=0UL; k<kend; k+=4UL ) {
1022  const size_t i1( element->index() );
1023  const ET1 v1( element->value() );
1024  ++element;
1025  const size_t i2( element->index() );
1026  const ET1 v2( element->value() );
1027  ++element;
1028  const size_t i3( element->index() );
1029  const ET1 v3( element->value() );
1030  ++element;
1031  const size_t i4( element->index() );
1032  const ET1 v4( element->value() );
1033  ++element;
1034 
1035  for( size_t j=0UL; j<jend; j+=4UL ) {
1036  (~C)(i,j ) -= v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1037  (~C)(i,j+1UL) -= v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1038  (~C)(i,j+2UL) -= v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1039  (~C)(i,j+3UL) -= v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1040  }
1041  for( size_t j=jend; j<B.columns(); ++j ) {
1042  (~C)(i,j) -= v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1043  }
1044  }
1045 
1046  for( ; element!=end; ++element ) {
1047  for( size_t j=0UL; j<jend; j+=4UL ) {
1048  (~C)(i,j ) -= element->value() * B(element->index(),j );
1049  (~C)(i,j+1UL) -= element->value() * B(element->index(),j+1UL);
1050  (~C)(i,j+2UL) -= element->value() * B(element->index(),j+2UL);
1051  (~C)(i,j+3UL) -= element->value() * B(element->index(),j+3UL);
1052  }
1053  for( size_t j=jend; j<B.columns(); ++j ) {
1054  (~C)(i,j) -= element->value() * B(element->index(),j);
1055  }
1056  }
1057  }
1058  }
1060  //**********************************************************************************************
1061 
1062  //**Vectorized subtraction assignment to dense matrices*****************************************
1076  template< typename MT3 // Type of the left-hand side target matrix
1077  , typename MT4 // Type of the left-hand side matrix operand
1078  , typename MT5 > // Type of the right-hand side matrix operand
1079  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
1080  selectSubAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
1081  {
1082  typedef IntrinsicTrait<ElementType> IT;
1083  typedef typename RemoveReference<LT>::Type::ConstIterator ConstIterator;
1084 
1085  const size_t N( B.columns() );
1086 
1087  for( size_t i=0UL; i<A.rows(); ++i )
1088  {
1089  const ConstIterator end( A.end(i) );
1090  ConstIterator element( A.begin(i) );
1091 
1092  const size_t kend( A.nonZeros(i) & size_t(-4) );
1093 
1094  for( size_t k=0UL; k<kend; k+=4UL ) {
1095  const size_t i1( element->index() );
1096  const IntrinsicType v1( set( element->value() ) );
1097  ++element;
1098  const size_t i2( element->index() );
1099  const IntrinsicType v2( set( element->value() ) );
1100  ++element;
1101  const size_t i3( element->index() );
1102  const IntrinsicType v3( set( element->value() ) );
1103  ++element;
1104  const size_t i4( element->index() );
1105  const IntrinsicType v4( set( element->value() ) );
1106  ++element;
1107 
1108  for( size_t j=0UL; j<N; j+=IT::size ) {
1109  store( &(~C)(i,j), load( &(~C)(i,j) ) - v1 * B.get(i1,j) - v2 * B.get(i2,j) - v3 * B.get(i3,j) - v4 * B.get(i4,j) );
1110  }
1111  }
1112 
1113  for( ; element!=end; ++element ) {
1114  const size_t i1( element->index() );
1115  const IntrinsicType v1( set( element->value() ) );
1116 
1117  for( size_t j=0UL; j<N; j+=IT::size ) {
1118  store( &(~C)(i,j), load( &(~C)(i,j) ) - v1 * B.get(i1,j) );
1119  }
1120  }
1121  }
1122  }
1124  //**********************************************************************************************
1125 
1126  //**Default subtraction assignment to column-major dense matrices*******************************
1140  template< typename MT3 // Type of the left-hand side target matrix
1141  , typename MT4 // Type of the left-hand side matrix operand
1142  , typename MT5 > // Type of the right-hand side matrix operand
1143  static inline void
1144  selectSubAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1145  {
1146  typedef typename RemoveReference<LT>::Type::ConstIterator ConstIterator;
1147 
1148  const size_t jend( B.columns() & size_t(-4) );
1149  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jend, "Invalid end calculation" );
1150 
1151  for( size_t j=0UL; j<jend; j+=4UL ) {
1152  for( size_t i=0UL; i<A.rows(); ++i )
1153  {
1154  ConstIterator element( A.begin(i) );
1155  const ConstIterator end( A.end(i) );
1156 
1157  for( ; element!=end; ++element ) {
1158  (~C)(i,j ) -= element->value() * B(element->index(),j );
1159  (~C)(i,j+1UL) -= element->value() * B(element->index(),j+1UL);
1160  (~C)(i,j+2UL) -= element->value() * B(element->index(),j+2UL);
1161  (~C)(i,j+3UL) -= element->value() * B(element->index(),j+3UL);
1162  }
1163  }
1164  }
1165 
1166  for( size_t j=jend; j<B.columns(); ++j ) {
1167  for( size_t i=0UL; i<A.rows(); ++i )
1168  {
1169  ConstIterator element( A.begin(i) );
1170  const ConstIterator end( A.end(i) );
1171 
1172  for( ; element!=end; ++element )
1173  (~C)(i,j) -= element->value() * B(element->index(),j);
1174  }
1175  }
1176  }
1178  //**********************************************************************************************
1179 
1180  //**Subtraction assignment to sparse matrices***************************************************
1181  // No special implementation for the subtraction assignment to sparse matrices.
1182  //**********************************************************************************************
1183 
1184  //**Multiplication assignment to dense matrices*************************************************
1185  // No special implementation for the multiplication assignment to dense matrices.
1186  //**********************************************************************************************
1187 
1188  //**Multiplication assignment to sparse matrices************************************************
1189  // No special implementation for the multiplication assignment to sparse matrices.
1190  //**********************************************************************************************
1191 
1192  //**Compile time checks*************************************************************************
1199  //**********************************************************************************************
1200 };
1201 //*************************************************************************************************
1202 
1203 
1204 
1205 
1206 //=================================================================================================
1207 //
1208 // GLOBAL BINARY ARITHMETIC OPERATORS
1209 //
1210 //=================================================================================================
1211 
1212 //*************************************************************************************************
1241 template< typename T1 // Type of the left-hand side sparse matrix
1242  , typename T2 > // Type of the right-hand side dense matrix
1243 inline const SMatDMatMultExpr<T1,T2>
1245 {
1247 
1248  if( (~lhs).columns() != (~rhs).rows() )
1249  throw std::invalid_argument( "Matrix sizes do not match" );
1250 
1251  return SMatDMatMultExpr<T1,T2>( ~lhs, ~rhs );
1252 }
1253 //*************************************************************************************************
1254 
1255 
1256 
1257 
1258 //=================================================================================================
1259 //
1260 // EXPRESSION TRAIT SPECIALIZATIONS
1261 //
1262 //=================================================================================================
1263 
1264 //*************************************************************************************************
1266 template< typename MT1, typename MT2, typename VT >
1267 struct DMatDVecMultExprTrait< SMatDMatMultExpr<MT1,MT2>, VT >
1268 {
1269  public:
1270  //**********************************************************************************************
1271  typedef typename SelectType< IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
1272  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value &&
1273  IsDenseVector<VT>::value && !IsTransposeVector<VT>::value
1274  , typename SMatDVecMultExprTrait< MT1, typename DMatDVecMultExprTrait<MT2,VT>::Type >::Type
1275  , INVALID_TYPE >::Type Type;
1276  //**********************************************************************************************
1277 };
1279 //*************************************************************************************************
1280 
1281 
1282 //*************************************************************************************************
1284 template< typename MT1, typename MT2, typename VT >
1285 struct DMatSVecMultExprTrait< SMatDMatMultExpr<MT1,MT2>, VT >
1286 {
1287  public:
1288  //**********************************************************************************************
1289  typedef typename SelectType< IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
1290  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value &&
1291  IsSparseVector<VT>::value && !IsTransposeVector<VT>::value
1292  , typename SMatDVecMultExprTrait< MT1, typename DMatSVecMultExprTrait<MT2,VT>::Type >::Type
1293  , INVALID_TYPE >::Type Type;
1294  //**********************************************************************************************
1295 };
1297 //*************************************************************************************************
1298 
1299 
1300 //*************************************************************************************************
1302 template< typename VT, typename MT1, typename MT2 >
1303 struct TDVecDMatMultExprTrait< VT, SMatDMatMultExpr<MT1,MT2> >
1304 {
1305  public:
1306  //**********************************************************************************************
1307  typedef typename SelectType< IsDenseVector<VT>::value && IsTransposeVector<VT>::value &&
1308  IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
1309  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value
1310  , typename TDVecDMatMultExprTrait< typename TDVecSMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
1311  , INVALID_TYPE >::Type Type;
1312  //**********************************************************************************************
1313 };
1315 //*************************************************************************************************
1316 
1317 
1318 //*************************************************************************************************
1320 template< typename VT, typename MT1, typename MT2 >
1321 struct TSVecDMatMultExprTrait< VT, SMatDMatMultExpr<MT1,MT2> >
1322 {
1323  public:
1324  //**********************************************************************************************
1325  typedef typename SelectType< IsSparseVector<VT>::value && IsTransposeVector<VT>::value &&
1326  IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
1327  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value
1328  , typename TSVecDMatMultExprTrait< typename TSVecSMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
1329  , INVALID_TYPE >::Type Type;
1330  //**********************************************************************************************
1331 };
1333 //*************************************************************************************************
1334 
1335 
1336 //*************************************************************************************************
1338 template< typename MT1, typename MT2 >
1339 struct RowExprTrait< SMatDMatMultExpr<MT1,MT2> >
1340 {
1341  public:
1342  //**********************************************************************************************
1343  typedef typename MultExprTrait< typename RowExprTrait<const MT1>::Type, MT2 >::Type Type;
1344  //**********************************************************************************************
1345 };
1347 //*************************************************************************************************
1348 
1349 
1350 //*************************************************************************************************
1352 template< typename MT1, typename MT2 >
1353 struct ColumnExprTrait< SMatDMatMultExpr<MT1,MT2> >
1354 {
1355  public:
1356  //**********************************************************************************************
1357  typedef typename MultExprTrait< MT1, typename ColumnExprTrait<const MT2>::Type >::Type Type;
1358  //**********************************************************************************************
1359 };
1361 //*************************************************************************************************
1362 
1363 } // namespace blaze
1364 
1365 #endif