All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
TDMatTSMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
20 //=================================================================================================
21 
22 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
23 #define _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
24 
25 
26 //*************************************************************************************************
27 // Includes
28 //*************************************************************************************************
29 
30 #include <stdexcept>
38 #include <blaze/math/Intrinsics.h>
40 #include <blaze/math/shims/Reset.h>
62 #include <blaze/util/Assert.h>
64 #include <blaze/util/DisableIf.h>
65 #include <blaze/util/EnableIf.h>
66 #include <blaze/util/InvalidType.h>
68 #include <blaze/util/SelectType.h>
69 #include <blaze/util/Types.h>
71 
72 
73 namespace blaze {
74 
75 //=================================================================================================
76 //
77 // CLASS TDMATTSMATMULTEXPR
78 //
79 //=================================================================================================
80 
81 //*************************************************************************************************
88 template< typename MT1 // Type of the left-hand side dense matrix
89  , typename MT2 > // Type of the right-hand side sparse matrix
90 class TDMatTSMatMultExpr : public DenseMatrix< TDMatTSMatMultExpr<MT1,MT2>, true >
91  , private MatMatMultExpr
92  , private Computation
93 {
94  private:
95  //**Type definitions****************************************************************************
96  typedef typename MT1::ResultType RT1;
97  typedef typename MT2::ResultType RT2;
98  typedef typename MT1::ElementType ET1;
99  typedef typename MT2::ElementType ET2;
100  typedef typename MT1::CompositeType CT1;
101  typedef typename MT2::CompositeType CT2;
102  //**********************************************************************************************
103 
104  //**********************************************************************************************
106 
107 
109  template< typename T1, typename T2, typename T3 >
110  struct UseVectorizedKernel {
111  enum { value = T1::vectorizable && T2::vectorizable &&
118  };
120  //**********************************************************************************************
121 
122  //**********************************************************************************************
124 
125 
128  template< typename T1, typename T2, typename T3 >
129  struct UseOptimizedKernel {
130  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
133  };
135  //**********************************************************************************************
136 
137  //**********************************************************************************************
139 
140 
142  template< typename T1, typename T2, typename T3 >
143  struct UseDefaultKernel {
144  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
145  !UseOptimizedKernel<T1,T2,T3>::value };
146  };
148  //**********************************************************************************************
149 
150  public:
151  //**Type definitions****************************************************************************
154  typedef typename ResultType::OppositeType OppositeType;
155  typedef typename ResultType::TransposeType TransposeType;
156  typedef typename ResultType::ElementType ElementType;
158  typedef const ElementType ReturnType;
159  typedef const ResultType CompositeType;
160 
162  typedef typename SelectType< IsExpression<MT1>::value, const MT1, const MT1& >::Type LeftOperand;
163 
165  typedef typename SelectType< IsExpression<MT2>::value, const MT2, const MT2& >::Type RightOperand;
166 
168  typedef typename SelectType< IsComputation<MT1>::value, const RT1, CT1 >::Type LT;
169 
171  typedef typename SelectType< IsComputation<MT2>::value, const RT2, CT2 >::Type RT;
172  //**********************************************************************************************
173 
174  //**Compilation flags***************************************************************************
176  enum { vectorizable = 0 };
177  //**********************************************************************************************
178 
179  //**Constructor*********************************************************************************
185  explicit inline TDMatTSMatMultExpr( const MT1& lhs, const MT2& rhs )
186  : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
187  , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
188  {
189  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
190  }
191  //**********************************************************************************************
192 
193  //**Access operator*****************************************************************************
200  inline ReturnType operator()( size_t i, size_t j ) const {
201  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
202  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
203 
204  typedef typename RemoveReference<CT2>::Type::ConstIterator ConstIterator;
205 
206  ElementType tmp = ElementType();
207 
208  // Early exit
209  if( lhs_.columns() == 0UL )
210  return tmp;
211 
212  // Fast computation in case the right-hand side sparse matrix directly provides iterators
214  {
215  CT2 B( rhs_ ); // Evaluation of the right-hand side sparse matrix operand
216 
217  const ConstIterator end( B.end(j) );
218  ConstIterator element( B.begin(j) );
219 
220  // Early exit in case column j is empty
221  if( element == end )
222  return tmp;
223 
224  // Calculating element (i,j)
225  tmp = lhs_(i,element->index()) * element->value();
226  ++element;
227  for( ; element!=end; ++element )
228  tmp += lhs_(i,element->index()) * element->value();
229  }
230 
231  // Default computation in case the right-hand side sparse matrix doesn't provide iterators
232  else {
233  tmp = lhs_(i,0UL) * rhs_(0UL,j);
234  for( size_t k=1UL; k<lhs_.columns(); ++k ) {
235  tmp += lhs_(i,k) * rhs_(k,j);
236  }
237  }
238 
239  return tmp;
240  }
241  //**********************************************************************************************
242 
243  //**Rows function*******************************************************************************
248  inline size_t rows() const {
249  return lhs_.rows();
250  }
251  //**********************************************************************************************
252 
253  //**Columns function****************************************************************************
258  inline size_t columns() const {
259  return rhs_.columns();
260  }
261  //**********************************************************************************************
262 
263  //**Left operand access*************************************************************************
268  inline LeftOperand leftOperand() const {
269  return lhs_;
270  }
271  //**********************************************************************************************
272 
273  //**Right operand access************************************************************************
278  inline RightOperand rightOperand() const {
279  return rhs_;
280  }
281  //**********************************************************************************************
282 
283  //**********************************************************************************************
289  template< typename T >
290  inline bool canAlias( const T* alias ) const {
291  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
292  }
293  //**********************************************************************************************
294 
295  //**********************************************************************************************
301  template< typename T >
302  inline bool isAliased( const T* alias ) const {
303  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
304  }
305  //**********************************************************************************************
306 
307  private:
308  //**Member variables****************************************************************************
311  //**********************************************************************************************
312 
313  //**Assignment to dense matrices****************************************************************
326  template< typename MT // Type of the target dense matrix
327  , bool SO > // Storage order of the target dense matrix
328  friend inline void assign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
329  {
331 
332  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
333  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
334 
335  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
336  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
337 
338  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
339  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
340  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
341  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
342  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
343  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
344 
345  TDMatTSMatMultExpr::selectAssignKernel( ~lhs, A, B );
346  }
348  //**********************************************************************************************
349 
350  //**Default assignment to row-major dense matrices**********************************************
364  template< typename MT3 // Type of the left-hand side target matrix
365  , typename MT4 // Type of the left-hand side matrix operand
366  , typename MT5 > // Type of the right-hand side matrix operand
367  static inline void
368  selectAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
369  {
370  typedef typename RemoveReference<RT>::Type::ConstIterator ConstIterator;
371 
372  const size_t iend( A.rows() & size_t(-4) );
373  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == iend, "Invalid end calculation" );
374 
375  for( size_t i=0UL; i<iend; i+=4UL ) {
376  for( size_t j=0UL; j<B.columns(); ++j )
377  {
378  ConstIterator element( B.begin(j) );
379  const ConstIterator end( B.end(j) );
380 
381  if( element == end ) {
382  reset( (~C)(i ,j) );
383  reset( (~C)(i+1UL,j) );
384  reset( (~C)(i+2UL,j) );
385  reset( (~C)(i+3UL,j) );
386  continue;
387  }
388 
389  (~C)(i ,j) = A(i ,element->index()) * element->value();
390  (~C)(i+1UL,j) = A(i+1UL,element->index()) * element->value();
391  (~C)(i+2UL,j) = A(i+2UL,element->index()) * element->value();
392  (~C)(i+3UL,j) = A(i+3UL,element->index()) * element->value();
393  ++element;
394  for( ; element!=end; ++element ) {
395  (~C)(i ,j) += A(i ,element->index()) * element->value();
396  (~C)(i+1UL,j) += A(i+1UL,element->index()) * element->value();
397  (~C)(i+2UL,j) += A(i+2UL,element->index()) * element->value();
398  (~C)(i+3UL,j) += A(i+3UL,element->index()) * element->value();
399  }
400  }
401  }
402 
403  for( size_t i=iend; i<A.rows(); ++i ) {
404  for( size_t j=0UL; j<B.columns(); ++j )
405  {
406  ConstIterator element( B.begin(j) );
407  const ConstIterator end( B.end(j) );
408 
409  if( element == end ) {
410  reset( (~C)(i,j) );
411  continue;
412  }
413 
414  (~C)(i,j) = A(i,element->index()) * element->value();
415  ++element;
416  for( ; element!=end; ++element )
417  (~C)(i,j) += A(i,element->index()) * element->value();
418  }
419  }
420  }
422  //**********************************************************************************************
423 
424  //**Default assignment to column-major dense matrices*******************************************
438  template< typename MT3 // Type of the left-hand side target matrix
439  , typename MT4 // Type of the left-hand side matrix operand
440  , typename MT5 > // Type of the right-hand side matrix operand
441  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
442  selectAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
443  {
444  typedef typename RemoveReference<RT>::Type::ConstIterator ConstIterator;
445 
446  for( size_t j=0UL; j<B.columns(); ++j ) {
447  for( size_t i=0UL; i<(~C).rows(); ++i ) {
448  reset( (~C)(i,j) );
449  }
450  ConstIterator element( B.begin(j) );
451  const ConstIterator end( B.end(j) );
452  for( ; element!=end; ++element ) {
453  for( size_t i=0UL; i<A.rows(); ++i ) {
454  if( isDefault( (~C)(i,j) ) )
455  (~C)(i,j) = A(i,element->index()) * element->value();
456  else
457  (~C)(i,j) += A(i,element->index()) * element->value();
458  }
459  }
460  }
461  }
463  //**********************************************************************************************
464 
465  //**Optimized assignment to column-major dense matrices*****************************************
479  template< typename MT3 // Type of the left-hand side target matrix
480  , typename MT4 // Type of the left-hand side matrix operand
481  , typename MT5 > // Type of the right-hand side matrix operand
482  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
483  selectAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
484  {
485  typedef typename RemoveReference<RT>::Type::ConstIterator ConstIterator;
486 
487  const size_t iend( A.rows() & size_t(-4) );
488  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == iend, "Invalid end calculation" );
489 
490  reset( ~C );
491 
492  for( size_t j=0UL; j<B.columns(); ++j )
493  {
494  const ConstIterator end( B.end(j) );
495  ConstIterator element( B.begin(j) );
496 
497  const size_t kend( B.nonZeros(j) & size_t(-4) );
498 
499  for( size_t k=0UL; k<kend; k+=4UL ) {
500  const size_t j1( element->index() );
501  const ET2 v1( element->value() );
502  ++element;
503  const size_t j2( element->index() );
504  const ET2 v2( element->value() );
505  ++element;
506  const size_t j3( element->index() );
507  const ET2 v3( element->value() );
508  ++element;
509  const size_t j4( element->index() );
510  const ET2 v4( element->value() );
511  ++element;
512 
513  for( size_t i=0UL; i<iend; i+=4UL ) {
514  (~C)(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
515  (~C)(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
516  (~C)(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
517  (~C)(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
518  }
519  for( size_t i=iend; i<A.rows(); ++i ) {
520  (~C)(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
521  }
522  }
523 
524  for( ; element!=end; ++element ) {
525  for( size_t i=0UL; i<iend; i+=4UL ) {
526  (~C)(i ,j) += A(i ,element->index()) * element->value();
527  (~C)(i+1UL,j) += A(i+1UL,element->index()) * element->value();
528  (~C)(i+2UL,j) += A(i+2UL,element->index()) * element->value();
529  (~C)(i+3UL,j) += A(i+3UL,element->index()) * element->value();
530  }
531  for( size_t i=iend; i<A.rows(); ++i ) {
532  (~C)(i,j) += A(i,element->index()) * element->value();
533  }
534  }
535  }
536  }
538  //**********************************************************************************************
539 
540  //**Vectorized assignment to column-major dense matrices****************************************
554  template< typename MT3 // Type of the left-hand side target matrix
555  , typename MT4 // Type of the left-hand side matrix operand
556  , typename MT5 > // Type of the right-hand side matrix operand
557  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
558  selectAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
559  {
560  typedef IntrinsicTrait<ElementType> IT;
561  typedef typename RemoveReference<RT>::Type::ConstIterator ConstIterator;
562 
563  const size_t M( A.rows() );
564 
565  reset( ~C );
566 
567  for( size_t j=0UL; j<B.columns(); ++j )
568  {
569  const ConstIterator end( B.end(j) );
570  ConstIterator element( B.begin(j) );
571 
572  const size_t kend( B.nonZeros(j) & size_t(-4) );
573 
574  for( size_t k=0UL; k<kend; k+=4UL ) {
575  const size_t j1( element->index() );
576  const IntrinsicType v1( set( element->value() ) );
577  ++element;
578  const size_t j2( element->index() );
579  const IntrinsicType v2( set( element->value() ) );
580  ++element;
581  const size_t j3( element->index() );
582  const IntrinsicType v3( set( element->value() ) );
583  ++element;
584  const size_t j4( element->index() );
585  const IntrinsicType v4( set( element->value() ) );
586  ++element;
587 
588  for( size_t i=0UL; i<M; i+=IT::size ) {
589  store( &(~C)(i,j), load( &(~C)(i,j) ) + A.get(i,j1) * v1 + A.get(i,j2) * v2 + A.get(i,j3) * v3 + A.get(i,j4) * v4 );
590  }
591  }
592 
593  for( ; element!=end; ++element ) {
594  const size_t j1( element->index() );
595  const IntrinsicType v1( set( element->value() ) );
596 
597  for( size_t i=0UL; i<M; i+=IT::size ) {
598  store( &(~C)(i,j), load( &(~C)(i,j) ) + A.get(i,j1) * v1 );
599  }
600  }
601  }
602  }
604  //**********************************************************************************************
605 
606  //**Assignment to sparse matrices***************************************************************
618  template< typename MT // Type of the target sparse matrix
619  , bool SO > // Storage order of the target sparse matrix
620  friend inline void assign( SparseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
621  {
623 
624  typedef typename SelectType< SO, ResultType, OppositeType >::Type TmpType;
625 
631  BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE( typename TmpType::CompositeType );
632 
633  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
634  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
635 
636  const TmpType tmp( rhs );
637  assign( ~lhs, tmp );
638  }
640  //**********************************************************************************************
641 
642  //**Addition assignment to dense matrices*******************************************************
655  template< typename MT // Type of the target dense matrix
656  , bool SO > // Storage order of the target dense matrix
657  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
658  {
660 
661  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
662  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
663 
664  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
665  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
666 
667  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
668  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
669  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
670  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
671  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
672  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
673 
674  TDMatTSMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
675  }
677  //**********************************************************************************************
678 
679  //**Default addition assignment to row-major dense matrices*************************************
693  template< typename MT3 // Type of the left-hand side target matrix
694  , typename MT4 // Type of the left-hand side matrix operand
695  , typename MT5 > // Type of the right-hand side matrix operand
696  static inline void
697  selectAddAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
698  {
699  typedef typename RemoveReference<RT>::Type::ConstIterator ConstIterator;
700 
701  const size_t iend( A.rows() & size_t(-4) );
702  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == iend, "Invalid end calculation" );
703 
704  for( size_t i=0UL; i<iend; i+=4UL ) {
705  for( size_t j=0UL; j<B.columns(); ++j )
706  {
707  ConstIterator element( B.begin(j) );
708  const ConstIterator end( B.end(j) );
709 
710  for( ; element!=end; ++element ) {
711  (~C)(i ,j) += A(i ,element->index()) * element->value();
712  (~C)(i+1UL,j) += A(i+1UL,element->index()) * element->value();
713  (~C)(i+2UL,j) += A(i+2UL,element->index()) * element->value();
714  (~C)(i+3UL,j) += A(i+3UL,element->index()) * element->value();
715  }
716  }
717  }
718 
719  for( size_t i=iend; i<A.rows(); ++i ) {
720  for( size_t j=0UL; j<B.columns(); ++j )
721  {
722  ConstIterator element( B.begin(j) );
723  const ConstIterator end( B.end(j) );
724 
725  for( ; element!=end; ++element )
726  (~C)(i,j) += A(i,element->index()) * element->value();
727  }
728  }
729  }
731  //**********************************************************************************************
732 
733  //**Default addition assignment to column-major dense matrices**********************************
747  template< typename MT3 // Type of the left-hand side target matrix
748  , typename MT4 // Type of the left-hand side matrix operand
749  , typename MT5 > // Type of the right-hand side matrix operand
750  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
751  selectAddAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
752  {
753  typedef typename RemoveReference<RT>::Type::ConstIterator ConstIterator;
754 
755  for( size_t j=0UL; j<B.columns(); ++j ) {
756  ConstIterator element( B.begin(j) );
757  const ConstIterator end( B.end(j) );
758  for( ; element!=end; ++element ) {
759  for( size_t i=0UL; i<A.rows(); ++i ) {
760  if( isDefault( (~C)(i,j) ) )
761  (~C)(i,j) = A(i,element->index()) * element->value();
762  else
763  (~C)(i,j) += A(i,element->index()) * element->value();
764  }
765  }
766  }
767  }
769  //**********************************************************************************************
770 
771  //**Optimized addition assignment to column-major dense matrices********************************
785  template< typename MT3 // Type of the left-hand side target matrix
786  , typename MT4 // Type of the left-hand side matrix operand
787  , typename MT5 > // Type of the right-hand side matrix operand
788  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
789  selectAddAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
790  {
791  typedef typename RemoveReference<RT>::Type::ConstIterator ConstIterator;
792 
793  const size_t iend( A.rows() & size_t(-4) );
794  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == iend, "Invalid end calculation" );
795 
796  for( size_t j=0UL; j<B.columns(); ++j )
797  {
798  const ConstIterator end( B.end(j) );
799  ConstIterator element( B.begin(j) );
800 
801  const size_t kend( B.nonZeros(j) & size_t(-4) );
802 
803  for( size_t k=0UL; k<kend; k+=4UL ) {
804  const size_t j1( element->index() );
805  const ET2 v1( element->value() );
806  ++element;
807  const size_t j2( element->index() );
808  const ET2 v2( element->value() );
809  ++element;
810  const size_t j3( element->index() );
811  const ET2 v3( element->value() );
812  ++element;
813  const size_t j4( element->index() );
814  const ET2 v4( element->value() );
815  ++element;
816 
817  for( size_t i=0UL; i<iend; i+=4UL ) {
818  (~C)(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
819  (~C)(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
820  (~C)(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
821  (~C)(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
822  }
823  for( size_t i=iend; i<A.rows(); ++i ) {
824  (~C)(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
825  }
826  }
827 
828  for( ; element!=end; ++element ) {
829  for( size_t i=0UL; i<iend; i+=4UL ) {
830  (~C)(i ,j) += A(i ,element->index()) * element->value();
831  (~C)(i+1UL,j) += A(i+1UL,element->index()) * element->value();
832  (~C)(i+2UL,j) += A(i+2UL,element->index()) * element->value();
833  (~C)(i+3UL,j) += A(i+3UL,element->index()) * element->value();
834  }
835  for( size_t i=iend; i<A.rows(); ++i ) {
836  (~C)(i,j) += A(i,element->index()) * element->value();
837  }
838  }
839  }
840  }
842  //**********************************************************************************************
843 
844  //**Vectorized addition assignment to column-major dense matrices*******************************
858  template< typename MT3 // Type of the left-hand side target matrix
859  , typename MT4 // Type of the left-hand side matrix operand
860  , typename MT5 > // Type of the right-hand side matrix operand
861  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
862  selectAddAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
863  {
864  typedef IntrinsicTrait<ElementType> IT;
865  typedef typename RemoveReference<RT>::Type::ConstIterator ConstIterator;
866 
867  const size_t M( A.rows() );
868 
869  for( size_t j=0UL; j<B.columns(); ++j )
870  {
871  const ConstIterator end( B.end(j) );
872  ConstIterator element( B.begin(j) );
873 
874  const size_t kend( B.nonZeros(j) & size_t(-4) );
875 
876  for( size_t k=0UL; k<kend; k+=4UL ) {
877  const size_t j1( element->index() );
878  const IntrinsicType v1( set( element->value() ) );
879  ++element;
880  const size_t j2( element->index() );
881  const IntrinsicType v2( set( element->value() ) );
882  ++element;
883  const size_t j3( element->index() );
884  const IntrinsicType v3( set( element->value() ) );
885  ++element;
886  const size_t j4( element->index() );
887  const IntrinsicType v4( set( element->value() ) );
888  ++element;
889 
890  for( size_t i=0UL; i<M; i+=IT::size ) {
891  store( &(~C)(i,j), load( &(~C)(i,j) ) + A.get(i,j1) * v1 + A.get(i,j2) * v2 + A.get(i,j3) * v3 + A.get(i,j4) * v4 );
892  }
893  }
894 
895  for( ; element!=end; ++element ) {
896  const size_t j1( element->index() );
897  const IntrinsicType v1( set( element->value() ) );
898 
899  for( size_t i=0UL; i<M; i+=IT::size ) {
900  store( &(~C)(i,j), load( &(~C)(i,j) ) + A.get(i,j1) * v1 );
901  }
902  }
903  }
904  }
906  //**********************************************************************************************
907 
908  //**Addition assignment to sparse matrices******************************************************
909  // No special implementation for the addition assignment to sparse matrices.
910  //**********************************************************************************************
911 
912  //**Subtraction assignment to dense matrices****************************************************
925  template< typename MT // Type of the target dense matrix
926  , bool SO > // Storage order of the target dense matrix
927  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
928  {
930 
931  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
932  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
933 
934  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
935  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
936 
937  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
938  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
939  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
940  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
941  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
942  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
943 
944  TDMatTSMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
945  }
947  //**********************************************************************************************
948 
949  //**Default subtraction assignment to row-major dense matrices**********************************
963  template< typename MT3 // Type of the left-hand side target matrix
964  , typename MT4 // Type of the left-hand side matrix operand
965  , typename MT5 > // Type of the right-hand side matrix operand
966  static inline void
967  selectSubAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
968  {
969  typedef typename RemoveReference<RT>::Type::ConstIterator ConstIterator;
970 
971  const size_t iend( A.rows() & size_t(-4) );
972  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == iend, "Invalid end calculation" );
973 
974  for( size_t i=0UL; i<iend; i+=4UL ) {
975  for( size_t j=0UL; j<B.columns(); ++j )
976  {
977  ConstIterator element( B.begin(j) );
978  const ConstIterator end( B.end(j) );
979 
980  for( ; element!=end; ++element ) {
981  (~C)(i ,j) -= A(i ,element->index()) * element->value();
982  (~C)(i+1UL,j) -= A(i+1UL,element->index()) * element->value();
983  (~C)(i+2UL,j) -= A(i+2UL,element->index()) * element->value();
984  (~C)(i+3UL,j) -= A(i+3UL,element->index()) * element->value();
985  }
986  }
987  }
988 
989  for( size_t i=iend; i<A.rows(); ++i ) {
990  for( size_t j=0UL; j<B.columns(); ++j )
991  {
992  ConstIterator element( B.begin(j) );
993  const ConstIterator end( B.end(j) );
994 
995  for( ; element!=end; ++element )
996  (~C)(i,j) -= A(i,element->index()) * element->value();
997  }
998  }
999  }
1001  //**********************************************************************************************
1002 
1003  //**Default subtraction assignment to column-major dense matrices*******************************
1017  template< typename MT3 // Type of the left-hand side target matrix
1018  , typename MT4 // Type of the left-hand side matrix operand
1019  , typename MT5 > // Type of the right-hand side matrix operand
1020  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
1021  selectSubAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1022  {
1023  typedef typename RemoveReference<RT>::Type::ConstIterator ConstIterator;
1024 
1025  for( size_t j=0UL; j<B.columns(); ++j ) {
1026  ConstIterator element( B.begin(j) );
1027  const ConstIterator end( B.end(j) );
1028  for( ; element!=end; ++element ) {
1029  for( size_t i=0UL; i<A.rows(); ++i ) {
1030  if( isDefault( (~C)(i,j) ) )
1031  (~C)(i,j) = -A(i,element->index()) * element->value();
1032  else
1033  (~C)(i,j) -= A(i,element->index()) * element->value();
1034  }
1035  }
1036  }
1037  }
1039  //**********************************************************************************************
1040 
1041  //**Optimized subtraction assignment to column-major dense matrices*****************************
1055  template< typename MT3 // Type of the left-hand side target matrix
1056  , typename MT4 // Type of the left-hand side matrix operand
1057  , typename MT5 > // Type of the right-hand side matrix operand
1058  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
1059  selectSubAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1060  {
1061  typedef typename RemoveReference<RT>::Type::ConstIterator ConstIterator;
1062 
1063  const size_t iend( A.rows() & size_t(-4) );
1064  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == iend, "Invalid end calculation" );
1065 
1066  for( size_t j=0UL; j<B.columns(); ++j )
1067  {
1068  const ConstIterator end( B.end(j) );
1069  ConstIterator element( B.begin(j) );
1070 
1071  const size_t kend( B.nonZeros(j) & size_t(-4) );
1072 
1073  for( size_t k=0UL; k<kend; k+=4UL ) {
1074  const size_t j1( element->index() );
1075  const ET2 v1( element->value() );
1076  ++element;
1077  const size_t j2( element->index() );
1078  const ET2 v2( element->value() );
1079  ++element;
1080  const size_t j3( element->index() );
1081  const ET2 v3( element->value() );
1082  ++element;
1083  const size_t j4( element->index() );
1084  const ET2 v4( element->value() );
1085  ++element;
1086 
1087  for( size_t i=0UL; i<iend; i+=4UL ) {
1088  (~C)(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1089  (~C)(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1090  (~C)(i+2UL,j) -= A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1091  (~C)(i+3UL,j) -= A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1092  }
1093  for( size_t i=iend; i<A.rows(); ++i ) {
1094  (~C)(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1095  }
1096  }
1097 
1098  for( ; element!=end; ++element ) {
1099  for( size_t i=0UL; i<iend; i+=4UL ) {
1100  (~C)(i ,j) -= A(i ,element->index()) * element->value();
1101  (~C)(i+1UL,j) -= A(i+1UL,element->index()) * element->value();
1102  (~C)(i+2UL,j) -= A(i+2UL,element->index()) * element->value();
1103  (~C)(i+3UL,j) -= A(i+3UL,element->index()) * element->value();
1104  }
1105  for( size_t i=iend; i<A.rows(); ++i ) {
1106  (~C)(i,j) -= A(i,element->index()) * element->value();
1107  }
1108  }
1109  }
1110  }
1112  //**********************************************************************************************
1113 
1114  //**Vectorized subtraction assignment to column-major dense matrices****************************
1128  template< typename MT3 // Type of the left-hand side target matrix
1129  , typename MT4 // Type of the left-hand side matrix operand
1130  , typename MT5 > // Type of the right-hand side matrix operand
1131  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
1132  selectSubAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1133  {
1134  typedef IntrinsicTrait<ElementType> IT;
1135  typedef typename RemoveReference<RT>::Type::ConstIterator ConstIterator;
1136 
1137  const size_t M( A.rows() );
1138 
1139  for( size_t j=0UL; j<B.columns(); ++j )
1140  {
1141  const ConstIterator end( B.end(j) );
1142  ConstIterator element( B.begin(j) );
1143 
1144  const size_t kend( B.nonZeros(j) & size_t(-4) );
1145 
1146  for( size_t k=0UL; k<kend; k+=4UL ) {
1147  const size_t j1( element->index() );
1148  const IntrinsicType v1( set( element->value() ) );
1149  ++element;
1150  const size_t j2( element->index() );
1151  const IntrinsicType v2( set( element->value() ) );
1152  ++element;
1153  const size_t j3( element->index() );
1154  const IntrinsicType v3( set( element->value() ) );
1155  ++element;
1156  const size_t j4( element->index() );
1157  const IntrinsicType v4( set( element->value() ) );
1158  ++element;
1159 
1160  for( size_t i=0UL; i<M; i+=IT::size ) {
1161  store( &(~C)(i,j), load( &(~C)(i,j) ) - A.get(i,j1) * v1 - A.get(i,j2) * v2 - A.get(i,j3) * v3 - A.get(i,j4) * v4 );
1162  }
1163  }
1164 
1165  for( ; element!=end; ++element ) {
1166  const size_t j1( element->index() );
1167  const IntrinsicType v1( set( element->value() ) );
1168 
1169  for( size_t i=0UL; i<M; i+=IT::size ) {
1170  store( &(~C)(i,j), load( &(~C)(i,j) ) - A.get(i,j1) * v1 );
1171  }
1172  }
1173  }
1174  }
1176  //**********************************************************************************************
1177 
1178  //**Subtraction assignment to sparse matrices***************************************************
1179  // No special implementation for the subtraction assignment to sparse matrices.
1180  //**********************************************************************************************
1181 
1182  //**Multiplication assignment to dense matrices*************************************************
1183  // No special implementation for the multiplication assignment to dense matrices.
1184  //**********************************************************************************************
1185 
1186  //**Multiplication assignment to sparse matrices************************************************
1187  // No special implementation for the multiplication assignment to sparse matrices.
1188  //**********************************************************************************************
1189 
1190  //**Compile time checks*************************************************************************
1197  //**********************************************************************************************
1198 };
1199 //*************************************************************************************************
1200 
1201 
1202 
1203 
1204 //=================================================================================================
1205 //
1206 // GLOBAL BINARY ARITHMETIC OPERATORS
1207 //
1208 //=================================================================================================
1209 
1210 //*************************************************************************************************
1239 template< typename T1 // Type of the left-hand side dense matrix
1240  , typename T2 > // Type of the right-hand side sparse matrix
1241 inline const TDMatTSMatMultExpr<T1,T2>
1243 {
1245 
1246  if( (~lhs).columns() != (~rhs).rows() )
1247  throw std::invalid_argument( "Matrix sizes do not match" );
1248 
1249  return TDMatTSMatMultExpr<T1,T2>( ~lhs, ~rhs );
1250 }
1251 //*************************************************************************************************
1252 
1253 
1254 
1255 
1256 //=================================================================================================
1257 //
1258 // EXPRESSION TRAIT SPECIALIZATIONS
1259 //
1260 //=================================================================================================
1261 
1262 //*************************************************************************************************
1264 template< typename MT1, typename MT2, typename VT >
1265 struct TDMatDVecMultExprTrait< TDMatTSMatMultExpr<MT1,MT2>, VT >
1266 {
1267  public:
1268  //**********************************************************************************************
1269  typedef typename SelectType< IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
1270  IsSparseMatrix<MT2>::value && IsColumnMajorMatrix<MT2>::value &&
1271  IsDenseVector<VT>::value && !IsTransposeVector<VT>::value
1272  , typename TDMatDVecMultExprTrait< MT1, typename TSMatDVecMultExprTrait<MT2,VT>::Type >::Type
1273  , INVALID_TYPE >::Type Type;
1274  //**********************************************************************************************
1275 };
1277 //*************************************************************************************************
1278 
1279 
1280 //*************************************************************************************************
1282 template< typename MT1, typename MT2, typename VT >
1283 struct TDMatSVecMultExprTrait< TDMatTSMatMultExpr<MT1,MT2>, VT >
1284 {
1285  public:
1286  //**********************************************************************************************
1287  typedef typename SelectType< IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
1288  IsSparseMatrix<MT2>::value && IsColumnMajorMatrix<MT2>::value &&
1289  IsSparseVector<VT>::value && !IsTransposeVector<VT>::value
1290  , typename TDMatDVecMultExprTrait< MT1, typename TSMatDVecMultExprTrait<MT2,VT>::Type >::Type
1291  , INVALID_TYPE >::Type Type;
1292  //**********************************************************************************************
1293 };
1295 //*************************************************************************************************
1296 
1297 
1298 //*************************************************************************************************
1300 template< typename VT, typename MT1, typename MT2 >
1301 struct TDVecTDMatMultExprTrait< VT, TDMatTSMatMultExpr<MT1,MT2> >
1302 {
1303  public:
1304  //**********************************************************************************************
1305  typedef typename SelectType< IsDenseVector<VT>::value && IsTransposeVector<VT>::value &&
1306  IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
1307  IsSparseMatrix<MT2>::value && IsColumnMajorMatrix<MT2>::value
1308  , typename TDVecTSMatMultExprTrait< typename TDVecTDMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
1309  , INVALID_TYPE >::Type Type;
1310  //**********************************************************************************************
1311 };
1313 //*************************************************************************************************
1314 
1315 
1316 //*************************************************************************************************
1318 template< typename VT, typename MT1, typename MT2 >
1319 struct TSVecTDMatMultExprTrait< VT, TDMatTSMatMultExpr<MT1,MT2> >
1320 {
1321  public:
1322  //**********************************************************************************************
1323  typedef typename SelectType< IsSparseVector<VT>::value && IsTransposeVector<VT>::value &&
1324  IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
1325  IsSparseMatrix<MT2>::value && IsColumnMajorMatrix<MT2>::value
1326  , typename TDVecTSMatMultExprTrait< typename TSVecTDMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
1327  , INVALID_TYPE >::Type Type;
1328  //**********************************************************************************************
1329 };
1331 //*************************************************************************************************
1332 
1333 
1334 //*************************************************************************************************
1336 template< typename MT1, typename MT2 >
1337 struct RowExprTrait< TDMatTSMatMultExpr<MT1,MT2> >
1338 {
1339  public:
1340  //**********************************************************************************************
1341  typedef typename MultExprTrait< typename RowExprTrait<const MT1>::Type, MT2 >::Type Type;
1342  //**********************************************************************************************
1343 };
1345 //*************************************************************************************************
1346 
1347 
1348 //*************************************************************************************************
1350 template< typename MT1, typename MT2 >
1351 struct ColumnExprTrait< TDMatTSMatMultExpr<MT1,MT2> >
1352 {
1353  public:
1354  //**********************************************************************************************
1355  typedef typename MultExprTrait< MT1, typename ColumnExprTrait<const MT2>::Type >::Type Type;
1356  //**********************************************************************************************
1357 };
1359 //*************************************************************************************************
1360 
1361 } // namespace blaze
1362 
1363 #endif