SMatDMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <stdexcept>
52 #include <blaze/math/Intrinsics.h>
53 #include <blaze/math/shims/Reset.h>
90 #include <blaze/util/Assert.h>
91 #include <blaze/util/DisableIf.h>
92 #include <blaze/util/EnableIf.h>
93 #include <blaze/util/InvalidType.h>
95 #include <blaze/util/mpl/And.h>
96 #include <blaze/util/mpl/Or.h>
97 #include <blaze/util/SelectType.h>
98 #include <blaze/util/Types.h>
102 
103 
104 namespace blaze {
105 
106 //=================================================================================================
107 //
108 // CLASS SMATDMATMULTEXPR
109 //
110 //=================================================================================================
111 
112 //*************************************************************************************************
119 template< typename MT1 // Type of the left-hand side sparse matrix
120  , typename MT2 > // Type of the right-hand side dense matrix
121 class SMatDMatMultExpr : public DenseMatrix< SMatDMatMultExpr<MT1,MT2>, false >
122  , private MatMatMultExpr
123  , private Computation
124 {
125  private:
126  //**Type definitions****************************************************************************
127  typedef typename MT1::ResultType RT1;
128  typedef typename MT2::ResultType RT2;
129  typedef typename RT1::ElementType ET1;
130  typedef typename RT2::ElementType ET2;
131  typedef typename MT1::CompositeType CT1;
132  typedef typename MT2::CompositeType CT2;
133  //**********************************************************************************************
134 
135  //**********************************************************************************************
138  //**********************************************************************************************
139 
140  //**********************************************************************************************
142  enum { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
143  //**********************************************************************************************
144 
145  //**********************************************************************************************
147 
153  template< typename T1, typename T2, typename T3 >
154  struct CanExploitSymmetry {
155  enum { value = IsColumnMajorMatrix<T1>::value &&
156  ( IsSymmetric<T2>::value || IsSymmetric<T3>::value ) };
157  };
159  //**********************************************************************************************
160 
161  //**********************************************************************************************
163 
167  template< typename T1, typename T2, typename T3 >
168  struct IsEvaluationRequired {
169  enum { value = ( evaluateLeft || evaluateRight ) &&
170  !CanExploitSymmetry<T1,T2,T3>::value };
171  };
173  //**********************************************************************************************
174 
175  //**********************************************************************************************
177 
180  template< typename T1, typename T2, typename T3 >
181  struct UseVectorizedKernel {
182  enum { value = !IsDiagonal<T3>::value &&
183  T1::vectorizable && T3::vectorizable &&
184  IsRowMajorMatrix<T1>::value &&
185  IsSame<typename T1::ElementType,typename T2::ElementType>::value &&
186  IsSame<typename T1::ElementType,typename T3::ElementType>::value &&
187  IntrinsicTrait<typename T1::ElementType>::addition &&
188  IntrinsicTrait<typename T1::ElementType>::subtraction &&
189  IntrinsicTrait<typename T1::ElementType>::multiplication };
190  };
192  //**********************************************************************************************
193 
194  //**********************************************************************************************
196 
200  template< typename T1, typename T2, typename T3 >
201  struct UseOptimizedKernel {
202  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
203  !IsDiagonal<T3>::value &&
204  !IsResizable<typename T1::ElementType>::value &&
205  !IsResizable<ET1>::value };
206  };
208  //**********************************************************************************************
209 
210  //**********************************************************************************************
212 
215  template< typename T1, typename T2, typename T3 >
216  struct UseDefaultKernel {
217  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
218  !UseOptimizedKernel<T1,T2,T3>::value };
219  };
221  //**********************************************************************************************
222 
223  public:
224  //**Type definitions****************************************************************************
231  typedef const ElementType ReturnType;
232  typedef const ResultType CompositeType;
233 
235  typedef typename SelectType< IsExpression<MT1>::value, const MT1, const MT1& >::Type LeftOperand;
236 
238  typedef typename SelectType< IsExpression<MT2>::value, const MT2, const MT2& >::Type RightOperand;
239 
242 
245  //**********************************************************************************************
246 
247  //**Compilation flags***************************************************************************
249  enum { vectorizable = !IsDiagonal<MT2>::value &&
250  MT2::vectorizable &&
254 
256  enum { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
257  !evaluateRight && MT2::smpAssignable };
258  //**********************************************************************************************
259 
260  //**Constructor*********************************************************************************
266  explicit inline SMatDMatMultExpr( const MT1& lhs, const MT2& rhs )
267  : lhs_( lhs ) // Left-hand side sparse matrix of the multiplication expression
268  , rhs_( rhs ) // Right-hand side dense matrix of the multiplication expression
269  {
270  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
271  }
272  //**********************************************************************************************
273 
274  //**Access operator*****************************************************************************
281  inline ReturnType operator()( size_t i, size_t j ) const {
282  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
283  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
284 
286 
287  ElementType tmp = ElementType();
288 
289  // Early exit
290  if( lhs_.columns() == 0UL )
291  return tmp;
292 
293  // Fast computation in case the left-hand side sparse matrix directly provides iterators
295  {
296  CT1 A( lhs_ ); // Evaluation of the left-hand side sparse matrix operand
297 
298  const ConstIterator end( ( IsUpper<MT2>::value )
299  ?( IsStrictlyUpper<MT2>::value ? A.lowerBound(i,j) : A.upperBound(i,j) )
300  :( A.end(i) ) );
301  ConstIterator element( ( IsLower<MT2>::value )
302  ?( IsStrictlyLower<MT2>::value ? A.upperBound(i,j) : A.lowerBound(i,j) )
303  :( A.begin(i) ) );
304 
305  if( element != end ) {
306  tmp = element->value() * rhs_(element->index(),j);
307  ++element;
308  for( ; element!=end; ++element ) {
309  tmp += element->value() * rhs_(element->index(),j);
310  }
311  }
312  }
313 
314  // Default computation in case the left-hand side sparse matrix doesn't provide iterators
315  else
316  {
317  const size_t kbegin( ( IsUpper<MT1>::value )
318  ?( ( IsLower<MT2>::value )
319  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
320  , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
321  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
322  :( ( IsLower<MT2>::value )
323  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
324  :( 0UL ) ) );
325  const size_t kend( ( IsLower<MT1>::value )
326  ?( ( IsUpper<MT2>::value )
327  ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
328  , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
329  :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
330  :( ( IsUpper<MT2>::value )
331  ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
332  :( lhs_.columns() ) ) );
333 
334  if( ( !IsTriangular<MT1>::value && !IsTriangular<MT2>::value ) || kbegin < kend ) {
335  tmp = lhs_(i,kbegin) * rhs_(kbegin,j);
336  for( size_t k=kbegin+1UL; k<kend; ++k ) {
337  tmp += lhs_(i,k) * rhs_(k,j);
338  }
339  }
340  }
341 
342  return tmp;
343  }
344  //**********************************************************************************************
345 
346  //**Rows function*******************************************************************************
351  inline size_t rows() const {
352  return lhs_.rows();
353  }
354  //**********************************************************************************************
355 
356  //**Columns function****************************************************************************
361  inline size_t columns() const {
362  return rhs_.columns();
363  }
364  //**********************************************************************************************
365 
366  //**Left operand access*************************************************************************
371  inline LeftOperand leftOperand() const {
372  return lhs_;
373  }
374  //**********************************************************************************************
375 
376  //**Right operand access************************************************************************
381  inline RightOperand rightOperand() const {
382  return rhs_;
383  }
384  //**********************************************************************************************
385 
386  //**********************************************************************************************
392  template< typename T >
393  inline bool canAlias( const T* alias ) const {
394  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
395  }
396  //**********************************************************************************************
397 
398  //**********************************************************************************************
404  template< typename T >
405  inline bool isAliased( const T* alias ) const {
406  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
407  }
408  //**********************************************************************************************
409 
410  //**********************************************************************************************
415  inline bool isAligned() const {
416  return rhs_.isAligned();
417  }
418  //**********************************************************************************************
419 
420  //**********************************************************************************************
425  inline bool canSMPAssign() const {
426  return ( rows() > SMP_SMATDMATMULT_THRESHOLD );
427  }
428  //**********************************************************************************************
429 
430  private:
431  //**Member variables****************************************************************************
432  LeftOperand lhs_;
433  RightOperand rhs_;
434  //**********************************************************************************************
435 
436  //**Assignment to dense matrices****************************************************************
449  template< typename MT // Type of the target dense matrix
450  , bool SO > // Storage order of the target dense matrix
451  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
452  assign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
453  {
455 
456  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
457  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
458 
459  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
460  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
461 
462  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
463  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
464  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
465  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
466  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
467  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
468 
469  SMatDMatMultExpr::selectAssignKernel( ~lhs, A, B );
470  }
472  //**********************************************************************************************
473 
474  //**Default assignment to row-major dense matrices**********************************************
488  template< typename MT3 // Type of the left-hand side target matrix
489  , typename MT4 // Type of the left-hand side matrix operand
490  , typename MT5 > // Type of the right-hand side matrix operand
491  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
492  selectAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
493  {
494  typedef typename MT4::ConstIterator ConstIterator;
495 
496  for( size_t i=0UL; i<A.rows(); ++i )
497  {
498  for( size_t j=0UL; j<(~C).columns(); ++j ) {
499  reset( (~C)(i,j) );
500  }
501 
502  ConstIterator element( A.begin(i) );
503  const ConstIterator end( A.end(i) );
504 
505  for( ; element!=end; ++element )
506  {
507  const size_t index( element->index() );
508 
509  if( IsDiagonal<MT5>::value )
510  {
511  (~C)(i,index) = element->value() * B(index,index);
512  }
513  else
514  {
515  const size_t jbegin( ( IsUpper<MT5>::value )
516  ?( IsStrictlyUpper<MT5>::value ? index+1UL : index )
517  :( 0UL ) );
518  const size_t jend( ( IsLower<MT5>::value )
519  ?( IsStrictlyLower<MT5>::value ? index : index+1UL )
520  :( B.columns() ) );
521  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
522 
523  for( size_t j=jbegin; j<jend; ++j ) {
524  if( isDefault( (~C)(i,j) ) )
525  (~C)(i,j) = element->value() * B(index,j);
526  else
527  (~C)(i,j) += element->value() * B(index,j);
528  }
529  }
530  }
531  }
532  }
534  //**********************************************************************************************
535 
536  //**Optimized assignment to row-major dense matrices********************************************
550  template< typename MT3 // Type of the left-hand side target matrix
551  , typename MT4 // Type of the left-hand side matrix operand
552  , typename MT5 > // Type of the right-hand side matrix operand
553  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
554  selectAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
555  {
556  typedef typename MT4::ConstIterator ConstIterator;
557 
558  reset( ~C );
559 
560  for( size_t i=0UL; i<A.rows(); ++i )
561  {
562  const ConstIterator end( A.end(i) );
563  ConstIterator element( A.begin(i) );
564 
565  const size_t nonzeros( A.nonZeros(i) );
566  const size_t kpos( nonzeros & size_t(-4) );
567  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
568 
569  for( size_t k=0UL; k<kpos; k+=4UL )
570  {
571  const size_t i1( element->index() );
572  const ET1 v1( element->value() );
573  ++element;
574  const size_t i2( element->index() );
575  const ET1 v2( element->value() );
576  ++element;
577  const size_t i3( element->index() );
578  const ET1 v3( element->value() );
579  ++element;
580  const size_t i4( element->index() );
581  const ET1 v4( element->value() );
582  ++element;
583 
584  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
585 
586  const size_t jbegin( ( IsUpper<MT5>::value )
587  ?( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 )
588  :( 0UL ) );
589  const size_t jend( ( IsLower<MT5>::value )
590  ?( IsStrictlyLower<MT5>::value ? i4 : i4+1UL )
591  :( B.columns() ) );
592  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
593 
594  const size_t jnum( jend - jbegin );
595  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
596  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
597 
598  for( size_t j=jbegin; j<jpos; j+=4UL ) {
599  (~C)(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
600  (~C)(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
601  (~C)(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
602  (~C)(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
603  }
604  for( size_t j=jpos; j<jend; ++j ) {
605  (~C)(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
606  }
607  }
608 
609  for( ; element!=end; ++element )
610  {
611  const size_t i1( element->index() );
612  const ET1 v1( element->value() );
613 
614  const size_t jbegin( ( IsUpper<MT5>::value )
615  ?( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 )
616  :( 0UL ) );
617  const size_t jend( ( IsLower<MT5>::value )
618  ?( IsStrictlyLower<MT5>::value ? i1 : i1+1UL )
619  :( B.columns() ) );
620  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
621 
622  const size_t jnum( jend - jbegin );
623  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
624  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
625 
626  for( size_t j=jbegin; j<jpos; j+=4UL ) {
627  (~C)(i,j ) += v1 * B(i1,j );
628  (~C)(i,j+1UL) += v1 * B(i1,j+1UL);
629  (~C)(i,j+2UL) += v1 * B(i1,j+2UL);
630  (~C)(i,j+3UL) += v1 * B(i1,j+3UL);
631  }
632  for( size_t j=jpos; j<jend; ++j ) {
633  (~C)(i,j) += v1 * B(i1,j);
634  }
635  }
636  }
637  }
639  //**********************************************************************************************
640 
641  //**Vectorized assignment to row-major dense matrices*******************************************
655  template< typename MT3 // Type of the left-hand side target matrix
656  , typename MT4 // Type of the left-hand side matrix operand
657  , typename MT5 > // Type of the right-hand side matrix operand
658  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
659  selectAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
660  {
661  typedef IntrinsicTrait<ElementType> IT;
662  typedef typename MT4::ConstIterator ConstIterator;
663 
664  reset( ~C );
665 
666  for( size_t i=0UL; i<A.rows(); ++i )
667  {
668  const ConstIterator end( A.end(i) );
669  ConstIterator element( A.begin(i) );
670 
671  const size_t nonzeros( A.nonZeros(i) );
672  const size_t kpos( nonzeros & size_t(-4) );
673  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
674 
675  for( size_t k=0UL; k<kpos; k+=4UL )
676  {
677  const size_t i1( element->index() );
678  const IntrinsicType v1( set( element->value() ) );
679  ++element;
680  const size_t i2( element->index() );
681  const IntrinsicType v2( set( element->value() ) );
682  ++element;
683  const size_t i3( element->index() );
684  const IntrinsicType v3( set( element->value() ) );
685  ++element;
686  const size_t i4( element->index() );
687  const IntrinsicType v4( set( element->value() ) );
688  ++element;
689 
690  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
691 
692  const size_t jbegin( ( IsUpper<MT5>::value )
693  ?( ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
694  :( 0UL ) );
695  const size_t jend( ( IsLower<MT5>::value )
696  ?( IsStrictlyLower<MT5>::value ? i4 : i4+1UL )
697  :( B.columns() ) );
698  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
699 
700  for( size_t j=jbegin; j<jend; j+=IT::size ) {
701  (~C).store( i, j, (~C).load(i,j) + v1 * B.load(i1,j) + v2 * B.load(i2,j) + v3 * B.load(i3,j) + v4 * B.load(i4,j) );
702  }
703  }
704 
705  for( ; element!=end; ++element )
706  {
707  const size_t i1( element->index() );
708  const IntrinsicType v1( set( element->value() ) );
709 
710  const size_t jbegin( ( IsUpper<MT5>::value )
711  ?( ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
712  :( 0UL ) );
713  const size_t jend( ( IsLower<MT5>::value )
714  ?( IsStrictlyLower<MT5>::value ? i1 : i1+1UL )
715  :( B.columns() ) );
716  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
717 
718  for( size_t j=jbegin; j<jend; j+=IT::size ) {
719  (~C).store( i, j, (~C).load(i,j) + v1 * B.load(i1,j) );
720  }
721  }
722  }
723  }
725  //**********************************************************************************************
726 
727  //**Default assignment to column-major dense matrices*******************************************
741  template< typename MT3 // Type of the left-hand side target matrix
742  , typename MT4 // Type of the left-hand side matrix operand
743  , typename MT5 > // Type of the right-hand side matrix operand
744  static inline void selectAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
745  {
746  typedef typename MT4::ConstIterator ConstIterator;
747 
748  const size_t jpos( B.columns() & size_t(-4) );
749  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jpos, "Invalid end calculation" );
750 
751  for( size_t j=0UL; j<jpos; j+=4UL ) {
752  for( size_t i=0UL; i<A.rows(); ++i )
753  {
754  ConstIterator element( ( IsLower<MT5>::value )
755  ?( IsStrictlyLower<MT5>::value ? A.upperBound(i,j) : A.lowerBound(i,j) )
756  :( A.begin(i) ) );
757  const ConstIterator end( ( IsUpper<MT5>::value )
758  ?( IsStrictlyUpper<MT5>::value ? A.lowerBound(i,j+4UL) : A.upperBound(i,j+4UL) )
759  :( A.end(i) ) );
760 
761  if( element == end ) {
762  reset( (~C)(i,j ) );
763  reset( (~C)(i,j+1UL) );
764  reset( (~C)(i,j+2UL) );
765  reset( (~C)(i,j+3UL) );
766  continue;
767  }
768 
769  (~C)(i,j ) = element->value() * B(element->index(),j );
770  (~C)(i,j+1UL) = element->value() * B(element->index(),j+1UL);
771  (~C)(i,j+2UL) = element->value() * B(element->index(),j+2UL);
772  (~C)(i,j+3UL) = element->value() * B(element->index(),j+3UL);
773  ++element;
774  for( ; element!=end; ++element ) {
775  (~C)(i,j ) += element->value() * B(element->index(),j );
776  (~C)(i,j+1UL) += element->value() * B(element->index(),j+1UL);
777  (~C)(i,j+2UL) += element->value() * B(element->index(),j+2UL);
778  (~C)(i,j+3UL) += element->value() * B(element->index(),j+3UL);
779  }
780  }
781  }
782 
783  for( size_t j=jpos; j<B.columns(); ++j ) {
784  for( size_t i=0UL; i<A.rows(); ++i )
785  {
786  ConstIterator element( ( IsLower<MT5>::value )
787  ?( IsStrictlyLower<MT5>::value ? A.upperBound(i,j) : A.lowerBound(i,j) )
788  :( A.begin(i) ) );
789  const ConstIterator end( ( IsUpper<MT5>::value )
790  ?( IsStrictlyUpper<MT5>::value ? A.lowerBound(i,j) : A.upperBound(i,j) )
791  :( A.end(i) ) );
792 
793  if( element == end ) {
794  reset( (~C)(i,j) );
795  continue;
796  }
797 
798  (~C)(i,j) = element->value() * B(element->index(),j);
799  ++element;
800  for( ; element!=end; ++element )
801  (~C)(i,j) += element->value() * B(element->index(),j);
802  }
803  }
804  }
806  //**********************************************************************************************
807 
808  //**Assignment to sparse matrices***************************************************************
821  template< typename MT // Type of the target sparse matrix
822  , bool SO > // Storage order of the target sparse matrix
823  friend inline void assign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
824  {
826 
827  typedef typename SelectType< SO, OppositeType, ResultType >::Type TmpType;
828 
835 
836  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
837  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
838 
839  const TmpType tmp( serial( rhs ) );
840  assign( ~lhs, tmp );
841  }
843  //**********************************************************************************************
844 
845  //**Restructuring assignment to column-major matrices*******************************************
860  template< typename MT > // Type of the target matrix
861  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
862  assign( Matrix<MT,true>& lhs, const SMatDMatMultExpr& rhs )
863  {
865 
867 
868  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
869  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
870 
871  if( IsSymmetric<MT1>::value )
872  assign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
873  else
874  assign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
875  }
877  //**********************************************************************************************
878 
879  //**Addition assignment to dense matrices*******************************************************
892  template< typename MT // Type of the target dense matrix
893  , bool SO > // Storage order of the target dense matrix
894  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
895  {
897 
898  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
899  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
900 
901  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
902  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
903 
904  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
905  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
906  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
907  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
908  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
909  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
910 
911  SMatDMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
912  }
914  //**********************************************************************************************
915 
916  //**Default addition assignment to row-major dense matrices*************************************
930  template< typename MT3 // Type of the left-hand side target matrix
931  , typename MT4 // Type of the left-hand side matrix operand
932  , typename MT5 > // Type of the right-hand side matrix operand
933  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
934  selectAddAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
935  {
936  typedef typename MT4::ConstIterator ConstIterator;
937 
938  for( size_t i=0UL; i<A.rows(); ++i )
939  {
940  const ConstIterator end( A.end(i) );
941  ConstIterator element( A.begin(i) );
942 
943  for( ; element!=end; ++element )
944  {
945  const size_t i1( element->index() );
946 
947  if( IsDiagonal<MT5>::value )
948  {
949  (~C)(i,i1) += element->value() * B(i1,i1);
950  }
951  else
952  {
953  const size_t jbegin( ( IsUpper<MT5>::value )
954  ?( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ):( 0UL ) );
955  const size_t jend( ( IsLower<MT5>::value )
956  ?( IsStrictlyLower<MT5>::value ? i1 : i1+1UL )
957  :( B.columns() ) );
958  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
959 
960  const size_t jnum( jend - jbegin );
961  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
962  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
963 
964  for( size_t j=jbegin; j<jpos; j+=4UL ) {
965  (~C)(i,j ) += element->value() * B(i1,j );
966  (~C)(i,j+1UL) += element->value() * B(i1,j+1UL);
967  (~C)(i,j+2UL) += element->value() * B(i1,j+2UL);
968  (~C)(i,j+3UL) += element->value() * B(i1,j+3UL);
969  }
970  for( size_t j=jpos; j<jend; ++j ) {
971  (~C)(i,j) += element->value() * B(i1,j);
972  }
973  }
974  }
975  }
976  }
978  //**********************************************************************************************
979 
980  //**Optimized addition assignment to row-major dense matrices***********************************
994  template< typename MT3 // Type of the left-hand side target matrix
995  , typename MT4 // Type of the left-hand side matrix operand
996  , typename MT5 > // Type of the right-hand side matrix operand
997  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
998  selectAddAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
999  {
1000  typedef typename MT4::ConstIterator ConstIterator;
1001 
1002  for( size_t i=0UL; i<A.rows(); ++i )
1003  {
1004  const ConstIterator end( A.end(i) );
1005  ConstIterator element( A.begin(i) );
1006 
1007  const size_t nonzeros( A.nonZeros(i) );
1008  const size_t kpos( nonzeros & size_t(-4) );
1009  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1010 
1011  for( size_t k=0UL; k<kpos; k+=4UL )
1012  {
1013  const size_t i1( element->index() );
1014  const ET1 v1( element->value() );
1015  ++element;
1016  const size_t i2( element->index() );
1017  const ET1 v2( element->value() );
1018  ++element;
1019  const size_t i3( element->index() );
1020  const ET1 v3( element->value() );
1021  ++element;
1022  const size_t i4( element->index() );
1023  const ET1 v4( element->value() );
1024  ++element;
1025 
1026  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1027 
1028  const size_t jbegin( ( IsUpper<MT5>::value )
1029  ?( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 )
1030  :( 0UL ) );
1031  const size_t jend( ( IsLower<MT5>::value )
1032  ?( IsStrictlyLower<MT5>::value ? i4 : i4+1UL )
1033  :( B.columns() ) );
1034  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1035 
1036  const size_t jnum( jend - jbegin );
1037  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1038  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1039 
1040  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1041  (~C)(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1042  (~C)(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1043  (~C)(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1044  (~C)(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1045  }
1046  for( size_t j=jpos; j<jend; ++j ) {
1047  (~C)(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1048  }
1049  }
1050 
1051  for( ; element!=end; ++element )
1052  {
1053  const size_t i1( element->index() );
1054  const ET1 v1( element->value() );
1055 
1056  const size_t jbegin( ( IsUpper<MT5>::value )
1057  ?( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 )
1058  :( 0UL ) );
1059  const size_t jend( ( IsLower<MT5>::value )
1060  ?( IsStrictlyLower<MT5>::value ? i1 : i1+1UL )
1061  :( B.columns() ) );
1062  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1063 
1064  const size_t jnum( jend - jbegin );
1065  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1066  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1067 
1068  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1069  (~C)(i,j ) += v1 * B(i1,j );
1070  (~C)(i,j+1UL) += v1 * B(i1,j+1UL);
1071  (~C)(i,j+2UL) += v1 * B(i1,j+2UL);
1072  (~C)(i,j+3UL) += v1 * B(i1,j+3UL);
1073  }
1074  for( size_t j=jpos; j<jend; ++j ) {
1075  (~C)(i,j) += v1 * B(i1,j);
1076  }
1077  }
1078  }
1079  }
1081  //**********************************************************************************************
1082 
1083  //**Vectorized addition assignment to row-major dense matrices**********************************
1097  template< typename MT3 // Type of the left-hand side target matrix
1098  , typename MT4 // Type of the left-hand side matrix operand
1099  , typename MT5 > // Type of the right-hand side matrix operand
1100  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
1101  selectAddAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
1102  {
1103  typedef IntrinsicTrait<ElementType> IT;
1104  typedef typename MT4::ConstIterator ConstIterator;
1105 
1106  for( size_t i=0UL; i<A.rows(); ++i )
1107  {
1108  const ConstIterator end( A.end(i) );
1109  ConstIterator element( A.begin(i) );
1110 
1111  const size_t nonzeros( A.nonZeros(i) );
1112  const size_t kpos( nonzeros & size_t(-4) );
1113  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1114 
1115  for( size_t k=0UL; k<kpos; k+=4UL )
1116  {
1117  const size_t i1( element->index() );
1118  const IntrinsicType v1( set( element->value() ) );
1119  ++element;
1120  const size_t i2( element->index() );
1121  const IntrinsicType v2( set( element->value() ) );
1122  ++element;
1123  const size_t i3( element->index() );
1124  const IntrinsicType v3( set( element->value() ) );
1125  ++element;
1126  const size_t i4( element->index() );
1127  const IntrinsicType v4( set( element->value() ) );
1128  ++element;
1129 
1130  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1131 
1132  const size_t jbegin( ( IsUpper<MT5>::value )
1133  ?( ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
1134  :( 0UL ) );
1135  const size_t jend( ( IsLower<MT5>::value )
1136  ?( IsStrictlyLower<MT5>::value ? i4 : i4+1UL )
1137  :( B.columns() ) );
1138  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1139 
1140  for( size_t j=jbegin; j<jend; j+=IT::size ) {
1141  (~C).store( i, j, (~C).load(i,j) + v1 * B.load(i1,j) + v2 * B.load(i2,j) + v3 * B.load(i3,j) + v4 * B.load(i4,j) );
1142  }
1143  }
1144 
1145  for( ; element!=end; ++element )
1146  {
1147  const size_t i1( element->index() );
1148  const IntrinsicType v1( set( element->value() ) );
1149 
1150  const size_t jbegin( ( IsUpper<MT5>::value )
1151  ?( ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
1152  :( 0UL ) );
1153  const size_t jend( ( IsLower<MT5>::value )
1154  ?( IsStrictlyLower<MT5>::value ? i1 : i1+1UL )
1155  :( B.columns() ) );
1156  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1157 
1158  for( size_t j=jbegin; j<jend; j+=IT::size ) {
1159  (~C).store( i, j, (~C).load(i,j) + v1 * B.load(i1,j) );
1160  }
1161  }
1162  }
1163  }
1165  //**********************************************************************************************
1166 
1167  //**Default addition assignment to column-major dense matrices**********************************
1181  template< typename MT3 // Type of the left-hand side target matrix
1182  , typename MT4 // Type of the left-hand side matrix operand
1183  , typename MT5 > // Type of the right-hand side matrix operand
1184  static inline void selectAddAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1185  {
1186  typedef typename MT4::ConstIterator ConstIterator;
1187 
1188  const size_t jpos( B.columns() & size_t(-4) );
1189  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jpos, "Invalid end calculation" );
1190 
1191  for( size_t j=0UL; j<jpos; j+=4UL ) {
1192  for( size_t i=0UL; i<A.rows(); ++i )
1193  {
1194  ConstIterator element( ( IsLower<MT5>::value )
1195  ?( IsStrictlyLower<MT5>::value ? A.upperBound(i,j) : A.lowerBound(i,j) )
1196  :( A.begin(i) ) );
1197  const ConstIterator end( ( IsUpper<MT5>::value )
1198  ?( IsStrictlyUpper<MT5>::value ? A.lowerBound(i,j+4UL) : A.upperBound(i,j+4UL) )
1199  :( A.end(i) ) );
1200 
1201  for( ; element!=end; ++element ) {
1202  (~C)(i,j ) += element->value() * B(element->index(),j );
1203  (~C)(i,j+1UL) += element->value() * B(element->index(),j+1UL);
1204  (~C)(i,j+2UL) += element->value() * B(element->index(),j+2UL);
1205  (~C)(i,j+3UL) += element->value() * B(element->index(),j+3UL);
1206  }
1207  }
1208  }
1209 
1210  for( size_t j=jpos; j<B.columns(); ++j ) {
1211  for( size_t i=0UL; i<A.rows(); ++i )
1212  {
1213  ConstIterator element( ( IsLower<MT5>::value )
1214  ?( IsStrictlyLower<MT5>::value ? A.upperBound(i,j) : A.lowerBound(i,j) )
1215  :( A.begin(i) ) );
1216  const ConstIterator end( ( IsUpper<MT5>::value )
1217  ?( IsStrictlyUpper<MT5>::value ? A.lowerBound(i,j) : A.upperBound(i,j) )
1218  :( A.end(i) ) );
1219 
1220  for( ; element!=end; ++element )
1221  (~C)(i,j) += element->value() * B(element->index(),j);
1222  }
1223  }
1224  }
1226  //**********************************************************************************************
1227 
1228  //**Restructuring addition assignment to column-major matrices**********************************
1243  template< typename MT > // Type of the target matrix
1244  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1245  addAssign( Matrix<MT,true>& lhs, const SMatDMatMultExpr& rhs )
1246  {
1248 
1250 
1251  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1252  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1253 
1254  if( IsSymmetric<MT1>::value )
1255  addAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1256  else
1257  addAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1258  }
1260  //**********************************************************************************************
1261 
1262  //**Addition assignment to sparse matrices******************************************************
1263  // No special implementation for the addition assignment to sparse matrices.
1264  //**********************************************************************************************
1265 
1266  //**Subtraction assignment to dense matrices****************************************************
1279  template< typename MT // Type of the target dense matrix
1280  , bool SO > // Storage order of the target dense matrix
1281  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1282  {
1284 
1285  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1286  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1287 
1288  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
1289  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
1290 
1291  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1292  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1293  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1294  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1295  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1296  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1297 
1298  SMatDMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1299  }
1301  //**********************************************************************************************
1302 
1303  //**Default subtraction assignment to row-major dense matrices**********************************
1317  template< typename MT3 // Type of the left-hand side target matrix
1318  , typename MT4 // Type of the left-hand side matrix operand
1319  , typename MT5 > // Type of the right-hand side matrix operand
1320  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
1321  selectSubAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
1322  {
1323  typedef typename MT4::ConstIterator ConstIterator;
1324 
1325  for( size_t i=0UL; i<A.rows(); ++i )
1326  {
1327  const ConstIterator end( A.end(i) );
1328  ConstIterator element( A.begin(i) );
1329 
1330  for( ; element!=end; ++element )
1331  {
1332  const size_t i1( element->index() );
1333 
1334  if( IsDiagonal<MT5>::value )
1335  {
1336  (~C)(i,i1) -= element->value() * B(i1,i1);
1337  }
1338  else
1339  {
1340  const size_t jbegin( ( IsUpper<MT5>::value )
1341  ?( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ):( 0UL ) );
1342  const size_t jend( ( IsLower<MT5>::value )
1343  ?( IsStrictlyLower<MT5>::value ? i1 : i1+1UL )
1344  :( B.columns() ) );
1345  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1346 
1347  const size_t jnum( jend - jbegin );
1348  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1349  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1350 
1351  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1352  (~C)(i,j ) -= element->value() * B(i1,j );
1353  (~C)(i,j+1UL) -= element->value() * B(i1,j+1UL);
1354  (~C)(i,j+2UL) -= element->value() * B(i1,j+2UL);
1355  (~C)(i,j+3UL) -= element->value() * B(i1,j+3UL);
1356  }
1357  for( size_t j=jpos; j<jend; ++j ) {
1358  (~C)(i,j) -= element->value() * B(i1,j);
1359  }
1360  }
1361  }
1362  }
1363  }
1365  //**********************************************************************************************
1366 
1367  //**Optimized subtraction assignment to row-major dense matrices********************************
1381  template< typename MT3 // Type of the left-hand side target matrix
1382  , typename MT4 // Type of the left-hand side matrix operand
1383  , typename MT5 > // Type of the right-hand side matrix operand
1384  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
1385  selectSubAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
1386  {
1387  typedef typename MT4::ConstIterator ConstIterator;
1388 
1389  for( size_t i=0UL; i<A.rows(); ++i )
1390  {
1391  const ConstIterator end( A.end(i) );
1392  ConstIterator element( A.begin(i) );
1393 
1394  const size_t nonzeros( A.nonZeros(i) );
1395  const size_t kpos( nonzeros & size_t(-4) );
1396  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1397 
1398  for( size_t k=0UL; k<kpos; k+=4UL )
1399  {
1400  const size_t i1( element->index() );
1401  const ET1 v1( element->value() );
1402  ++element;
1403  const size_t i2( element->index() );
1404  const ET1 v2( element->value() );
1405  ++element;
1406  const size_t i3( element->index() );
1407  const ET1 v3( element->value() );
1408  ++element;
1409  const size_t i4( element->index() );
1410  const ET1 v4( element->value() );
1411  ++element;
1412 
1413  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1414 
1415  const size_t jbegin( ( IsUpper<MT5>::value )
1416  ?( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 )
1417  :( 0UL ) );
1418  const size_t jend( ( IsLower<MT5>::value )
1419  ?( IsStrictlyLower<MT5>::value ? i4 : i4+1UL )
1420  :( B.columns() ) );
1421  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1422 
1423  const size_t jnum( jend - jbegin );
1424  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1425  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1426 
1427  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1428  (~C)(i,j ) -= v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1429  (~C)(i,j+1UL) -= v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1430  (~C)(i,j+2UL) -= v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1431  (~C)(i,j+3UL) -= v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1432  }
1433  for( size_t j=jpos; j<jend; ++j ) {
1434  (~C)(i,j) -= v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1435  }
1436  }
1437 
1438  for( ; element!=end; ++element )
1439  {
1440  const size_t i1( element->index() );
1441  const ET1 v1( element->value() );
1442 
1443  const size_t jbegin( ( IsUpper<MT5>::value )
1444  ?( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 )
1445  :( 0UL ) );
1446  const size_t jend( ( IsLower<MT5>::value )
1447  ?( IsStrictlyLower<MT5>::value ? i1 : i1+1UL )
1448  :( B.columns() ) );
1449  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1450 
1451  const size_t jnum( jend - jbegin );
1452  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1453  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1454 
1455  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1456  (~C)(i,j ) -= v1 * B(i1,j );
1457  (~C)(i,j+1UL) -= v1 * B(i1,j+1UL);
1458  (~C)(i,j+2UL) -= v1 * B(i1,j+2UL);
1459  (~C)(i,j+3UL) -= v1 * B(i1,j+3UL);
1460  }
1461  for( size_t j=jpos; j<jend; ++j ) {
1462  (~C)(i,j) -= v1 * B(i1,j);
1463  }
1464  }
1465  }
1466  }
1468  //**********************************************************************************************
1469 
1470  //**Vectorized subtraction assignment to row-major dense matrices*******************************
1484  template< typename MT3 // Type of the left-hand side target matrix
1485  , typename MT4 // Type of the left-hand side matrix operand
1486  , typename MT5 > // Type of the right-hand side matrix operand
1487  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
1488  selectSubAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
1489  {
1490  typedef IntrinsicTrait<ElementType> IT;
1491  typedef typename MT4::ConstIterator ConstIterator;
1492 
1493  for( size_t i=0UL; i<A.rows(); ++i )
1494  {
1495  const ConstIterator end( A.end(i) );
1496  ConstIterator element( A.begin(i) );
1497 
1498  const size_t nonzeros( A.nonZeros(i) );
1499  const size_t kpos( nonzeros & size_t(-4) );
1500  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1501 
1502  for( size_t k=0UL; k<kpos; k+=4UL )
1503  {
1504  const size_t i1( element->index() );
1505  const IntrinsicType v1( set( element->value() ) );
1506  ++element;
1507  const size_t i2( element->index() );
1508  const IntrinsicType v2( set( element->value() ) );
1509  ++element;
1510  const size_t i3( element->index() );
1511  const IntrinsicType v3( set( element->value() ) );
1512  ++element;
1513  const size_t i4( element->index() );
1514  const IntrinsicType v4( set( element->value() ) );
1515  ++element;
1516 
1517  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1518 
1519  const size_t jbegin( ( IsUpper<MT5>::value )
1520  ?( ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
1521  :( 0UL ) );
1522  const size_t jend( ( IsLower<MT5>::value )
1523  ?( IsStrictlyLower<MT5>::value ? i4 : i4+1UL )
1524  :( B.columns() ) );
1525  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1526 
1527  for( size_t j=jbegin; j<jend; j+=IT::size ) {
1528  (~C).store( i, j, (~C).load(i,j) - v1 * B.load(i1,j) - v2 * B.load(i2,j) - v3 * B.load(i3,j) - v4 * B.load(i4,j) );
1529  }
1530  }
1531 
1532  for( ; element!=end; ++element )
1533  {
1534  const size_t i1( element->index() );
1535  const IntrinsicType v1( set( element->value() ) );
1536 
1537  const size_t jbegin( ( IsUpper<MT5>::value )
1538  ?( ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
1539  :( 0UL ) );
1540  const size_t jend( ( IsLower<MT5>::value )
1541  ?( IsStrictlyLower<MT5>::value ? i1 : i1+1UL )
1542  :( B.columns() ) );
1543  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1544 
1545  for( size_t j=jbegin; j<jend; j+=IT::size ) {
1546  (~C).store( i, j, (~C).load(i,j) - v1 * B.load(i1,j) );
1547  }
1548  }
1549  }
1550  }
1552  //**********************************************************************************************
1553 
1554  //**Default subtraction assignment to column-major dense matrices*******************************
1568  template< typename MT3 // Type of the left-hand side target matrix
1569  , typename MT4 // Type of the left-hand side matrix operand
1570  , typename MT5 > // Type of the right-hand side matrix operand
1571  static inline void selectSubAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1572  {
1573  typedef typename MT4::ConstIterator ConstIterator;
1574 
1575  const size_t jpos( B.columns() & size_t(-4) );
1576  BLAZE_INTERNAL_ASSERT( ( B.columns() - ( B.columns() % 4UL ) ) == jpos, "Invalid end calculation" );
1577 
1578  for( size_t j=0UL; j<jpos; j+=4UL ) {
1579  for( size_t i=0UL; i<A.rows(); ++i )
1580  {
1581  ConstIterator element( ( IsLower<MT5>::value )
1582  ?( IsStrictlyLower<MT5>::value ? A.upperBound(i,j) : A.lowerBound(i,j) )
1583  :( A.begin(i) ) );
1584  const ConstIterator end( ( IsUpper<MT5>::value )
1585  ?( IsStrictlyUpper<MT5>::value ? A.lowerBound(i,j+4UL) : A.upperBound(i,j+4UL) )
1586  :( A.end(i) ) );
1587 
1588  for( ; element!=end; ++element ) {
1589  (~C)(i,j ) -= element->value() * B(element->index(),j );
1590  (~C)(i,j+1UL) -= element->value() * B(element->index(),j+1UL);
1591  (~C)(i,j+2UL) -= element->value() * B(element->index(),j+2UL);
1592  (~C)(i,j+3UL) -= element->value() * B(element->index(),j+3UL);
1593  }
1594  }
1595  }
1596 
1597  for( size_t j=jpos; j<B.columns(); ++j ) {
1598  for( size_t i=0UL; i<A.rows(); ++i )
1599  {
1600  ConstIterator element( ( IsLower<MT5>::value )
1601  ?( IsStrictlyLower<MT5>::value ? A.upperBound(i,j) : A.lowerBound(i,j) )
1602  :( A.begin(i) ) );
1603  const ConstIterator end( ( IsUpper<MT5>::value )
1604  ?( IsStrictlyUpper<MT5>::value ? A.lowerBound(i,j) : A.upperBound(i,j) )
1605  :( A.end(i) ) );
1606 
1607  for( ; element!=end; ++element )
1608  (~C)(i,j) -= element->value() * B(element->index(),j);
1609  }
1610  }
1611  }
1613  //**********************************************************************************************
1614 
1615  //**Restructuring subtraction assignment to column-major matrices*******************************
1630  template< typename MT > // Type of the target matrix
1631  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1632  subAssign( Matrix<MT,true>& lhs, const SMatDMatMultExpr& rhs )
1633  {
1635 
1637 
1638  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1639  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1640 
1641  if( IsSymmetric<MT1>::value )
1642  subAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1643  else
1644  subAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1645  }
1647  //**********************************************************************************************
1648 
1649  //**Subtraction assignment to sparse matrices***************************************************
1650  // No special implementation for the subtraction assignment to sparse matrices.
1651  //**********************************************************************************************
1652 
1653  //**Multiplication assignment to dense matrices*************************************************
1654  // No special implementation for the multiplication assignment to dense matrices.
1655  //**********************************************************************************************
1656 
1657  //**Multiplication assignment to sparse matrices************************************************
1658  // No special implementation for the multiplication assignment to sparse matrices.
1659  //**********************************************************************************************
1660 
1661  //**SMP assignment to dense matrices************************************************************
1676  template< typename MT // Type of the target dense matrix
1677  , bool SO > // Storage order of the target dense matrix
1678  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1679  smpAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1680  {
1682 
1683  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1684  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1685 
1686  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1687  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1688 
1689  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1690  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1691  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1692  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1693  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1694  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1695 
1696  smpAssign( ~lhs, A * B );
1697  }
1699  //**********************************************************************************************
1700 
1701  //**SMP assignment to sparse matrices***********************************************************
1716  template< typename MT // Type of the target sparse matrix
1717  , bool SO > // Storage order of the target sparse matrix
1718  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1719  smpAssign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1720  {
1722 
1723  typedef typename SelectType< SO, OppositeType, ResultType >::Type TmpType;
1724 
1731 
1732  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1733  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1734 
1735  const TmpType tmp( rhs );
1736  smpAssign( ~lhs, tmp );
1737  }
1739  //**********************************************************************************************
1740 
1741  //**Restructuring SMP assignment to column-major matrices***************************************
1756  template< typename MT > // Type of the target matrix
1757  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1758  smpAssign( Matrix<MT,true>& lhs, const SMatDMatMultExpr& rhs )
1759  {
1761 
1763 
1764  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1765  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1766 
1767  if( IsSymmetric<MT1>::value )
1768  smpAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1769  else
1770  smpAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1771  }
1773  //**********************************************************************************************
1774 
1775  //**SMP addition assignment to dense matrices***************************************************
1791  template< typename MT // Type of the target dense matrix
1792  , bool SO > // Storage order of the target dense matrix
1793  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1794  smpAddAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1795  {
1797 
1798  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1799  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1800 
1801  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1802  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1803 
1804  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1805  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1806  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1807  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1808  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1809  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1810 
1811  smpAddAssign( ~lhs, A * B );
1812  }
1814  //**********************************************************************************************
1815 
1816  //**Restructuring SMP addition assignment to column-major matrices******************************
1831  template< typename MT > // Type of the target matrix
1832  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1833  smpAddAssign( Matrix<MT,true>& lhs, const SMatDMatMultExpr& rhs )
1834  {
1836 
1838 
1839  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1840  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1841 
1842  if( IsSymmetric<MT1>::value )
1843  smpAddAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1844  else
1845  smpAddAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1846  }
1848  //**********************************************************************************************
1849 
1850  //**SMP addition assignment to sparse matrices**************************************************
1851  // No special implementation for the SMP addition assignment to sparse matrices.
1852  //**********************************************************************************************
1853 
1854  //**SMP subtraction assignment to dense matrices************************************************
1870  template< typename MT // Type of the target dense matrix
1871  , bool SO > // Storage order of the target dense matrix
1872  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1873  smpSubAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1874  {
1876 
1877  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1878  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1879 
1880  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1881  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1882 
1883  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1884  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1885  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1886  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1887  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1888  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1889 
1890  smpSubAssign( ~lhs, A * B );
1891  }
1893  //**********************************************************************************************
1894 
1895  //**Restructuring SMP subtraction assignment to column-major matrices***************************
1910  template< typename MT > // Type of the target matrix
1911  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1912  smpSubAssign( Matrix<MT,true>& lhs, const SMatDMatMultExpr& rhs )
1913  {
1915 
1917 
1918  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1919  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1920 
1921  if( IsSymmetric<MT1>::value )
1922  smpSubAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1923  else
1924  smpSubAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1925  }
1927  //**********************************************************************************************
1928 
1929  //**SMP subtraction assignment to sparse matrices***********************************************
1930  // No special implementation for the SMP subtraction assignment to sparse matrices.
1931  //**********************************************************************************************
1932 
1933  //**SMP multiplication assignment to dense matrices*********************************************
1934  // No special implementation for the SMP multiplication assignment to dense matrices.
1935  //**********************************************************************************************
1936 
1937  //**SMP multiplication assignment to sparse matrices********************************************
1938  // No special implementation for the SMP multiplication assignment to sparse matrices.
1939  //**********************************************************************************************
1940 
1941  //**Compile time checks*************************************************************************
1949  //**********************************************************************************************
1950 };
1951 //*************************************************************************************************
1952 
1953 
1954 
1955 
1956 //=================================================================================================
1957 //
1958 // GLOBAL BINARY ARITHMETIC OPERATORS
1959 //
1960 //=================================================================================================
1961 
1962 //*************************************************************************************************
1991 template< typename T1 // Type of the left-hand side sparse matrix
1992  , typename T2 > // Type of the right-hand side dense matrix
1993 inline const SMatDMatMultExpr<T1,T2>
1995 {
1997 
1998  if( (~lhs).columns() != (~rhs).rows() )
1999  throw std::invalid_argument( "Matrix sizes do not match" );
2000 
2001  return SMatDMatMultExpr<T1,T2>( ~lhs, ~rhs );
2002 }
2003 //*************************************************************************************************
2004 
2005 
2006 
2007 
2008 //=================================================================================================
2009 //
2010 // ROWS SPECIALIZATIONS
2011 //
2012 //=================================================================================================
2013 
2014 //*************************************************************************************************
2016 template< typename MT1, typename MT2 >
2017 struct Rows< SMatDMatMultExpr<MT1,MT2> >
2018  : public Rows<MT1>
2019 {};
2021 //*************************************************************************************************
2022 
2023 
2024 
2025 
2026 //=================================================================================================
2027 //
2028 // COLUMNS SPECIALIZATIONS
2029 //
2030 //=================================================================================================
2031 
2032 //*************************************************************************************************
2034 template< typename MT1, typename MT2 >
2035 struct Columns< SMatDMatMultExpr<MT1,MT2> >
2036  : public Columns<MT2>
2037 {};
2039 //*************************************************************************************************
2040 
2041 
2042 
2043 
2044 //=================================================================================================
2045 //
2046 // ISLOWER SPECIALIZATIONS
2047 //
2048 //=================================================================================================
2049 
2050 //*************************************************************************************************
2052 template< typename MT1, typename MT2 >
2053 struct IsLower< SMatDMatMultExpr<MT1,MT2> >
2054  : public IsTrue< And< IsLower<MT1>, IsLower<MT2> >::value >
2055 {};
2057 //*************************************************************************************************
2058 
2059 
2060 
2061 
2062 //=================================================================================================
2063 //
2064 // ISUNILOWER SPECIALIZATIONS
2065 //
2066 //=================================================================================================
2067 
2068 //*************************************************************************************************
2070 template< typename MT1, typename MT2 >
2071 struct IsUniLower< SMatDMatMultExpr<MT1,MT2> >
2072  : public IsTrue< And< IsUniLower<MT1>, IsUniLower<MT2> >::value >
2073 {};
2075 //*************************************************************************************************
2076 
2077 
2078 
2079 
2080 //=================================================================================================
2081 //
2082 // ISSTRICTLYLOWER SPECIALIZATIONS
2083 //
2084 //=================================================================================================
2085 
2086 //*************************************************************************************************
2088 template< typename MT1, typename MT2 >
2089 struct IsStrictlyLower< SMatDMatMultExpr<MT1,MT2> >
2090  : public IsTrue< Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2091  , And< IsStrictlyLower<MT2>, IsLower<MT1> > >::value >
2092 {};
2094 //*************************************************************************************************
2095 
2096 
2097 
2098 
2099 //=================================================================================================
2100 //
2101 // ISUPPER SPECIALIZATIONS
2102 //
2103 //=================================================================================================
2104 
2105 //*************************************************************************************************
2107 template< typename MT1, typename MT2 >
2108 struct IsUpper< SMatDMatMultExpr<MT1,MT2> >
2109  : public IsTrue< And< IsUpper<MT1>, IsUpper<MT2> >::value >
2110 {};
2112 //*************************************************************************************************
2113 
2114 
2115 
2116 
2117 //=================================================================================================
2118 //
2119 // ISUNIUPPER SPECIALIZATIONS
2120 //
2121 //=================================================================================================
2122 
2123 //*************************************************************************************************
2125 template< typename MT1, typename MT2 >
2126 struct IsUniUpper< SMatDMatMultExpr<MT1,MT2> >
2127  : public IsTrue< And< IsUniUpper<MT1>, IsUniUpper<MT2> >::value >
2128 {};
2130 //*************************************************************************************************
2131 
2132 
2133 
2134 
2135 //=================================================================================================
2136 //
2137 // ISSTRICTLYUPPER SPECIALIZATIONS
2138 //
2139 //=================================================================================================
2140 
2141 //*************************************************************************************************
2143 template< typename MT1, typename MT2 >
2144 struct IsStrictlyUpper< SMatDMatMultExpr<MT1,MT2> >
2145  : public IsTrue< Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2146  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > >::value >
2147 {};
2149 //*************************************************************************************************
2150 
2151 
2152 
2153 
2154 //=================================================================================================
2155 //
2156 // EXPRESSION TRAIT SPECIALIZATIONS
2157 //
2158 //=================================================================================================
2159 
2160 //*************************************************************************************************
2162 template< typename MT1, typename MT2, typename VT >
2163 struct DMatDVecMultExprTrait< SMatDMatMultExpr<MT1,MT2>, VT >
2164 {
2165  public:
2166  //**********************************************************************************************
2167  typedef typename SelectType< IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
2168  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value &&
2169  IsDenseVector<VT>::value && IsColumnVector<VT>::value
2170  , typename SMatDVecMultExprTrait< MT1, typename DMatDVecMultExprTrait<MT2,VT>::Type >::Type
2171  , INVALID_TYPE >::Type Type;
2172  //**********************************************************************************************
2173 };
2175 //*************************************************************************************************
2176 
2177 
2178 //*************************************************************************************************
2180 template< typename MT1, typename MT2, typename VT >
2181 struct DMatSVecMultExprTrait< SMatDMatMultExpr<MT1,MT2>, VT >
2182 {
2183  public:
2184  //**********************************************************************************************
2185  typedef typename SelectType< IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
2186  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value &&
2187  IsSparseVector<VT>::value && IsColumnVector<VT>::value
2188  , typename SMatDVecMultExprTrait< MT1, typename DMatSVecMultExprTrait<MT2,VT>::Type >::Type
2189  , INVALID_TYPE >::Type Type;
2190  //**********************************************************************************************
2191 };
2193 //*************************************************************************************************
2194 
2195 
2196 //*************************************************************************************************
2198 template< typename VT, typename MT1, typename MT2 >
2199 struct TDVecDMatMultExprTrait< VT, SMatDMatMultExpr<MT1,MT2> >
2200 {
2201  public:
2202  //**********************************************************************************************
2203  typedef typename SelectType< IsDenseVector<VT>::value && IsRowVector<VT>::value &&
2204  IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
2205  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value
2206  , typename TDVecDMatMultExprTrait< typename TDVecSMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
2207  , INVALID_TYPE >::Type Type;
2208  //**********************************************************************************************
2209 };
2211 //*************************************************************************************************
2212 
2213 
2214 //*************************************************************************************************
2216 template< typename VT, typename MT1, typename MT2 >
2217 struct TSVecDMatMultExprTrait< VT, SMatDMatMultExpr<MT1,MT2> >
2218 {
2219  public:
2220  //**********************************************************************************************
2221  typedef typename SelectType< IsSparseVector<VT>::value && IsRowVector<VT>::value &&
2222  IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
2223  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value
2224  , typename TSVecDMatMultExprTrait< typename TSVecSMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
2225  , INVALID_TYPE >::Type Type;
2226  //**********************************************************************************************
2227 };
2229 //*************************************************************************************************
2230 
2231 
2232 //*************************************************************************************************
2234 template< typename MT1, typename MT2, bool AF >
2235 struct SubmatrixExprTrait< SMatDMatMultExpr<MT1,MT2>, AF >
2236 {
2237  public:
2238  //**********************************************************************************************
2239  typedef typename MultExprTrait< typename SubmatrixExprTrait<const MT1,AF>::Type
2240  , typename SubmatrixExprTrait<const MT2,AF>::Type >::Type Type;
2241  //**********************************************************************************************
2242 };
2244 //*************************************************************************************************
2245 
2246 
2247 //*************************************************************************************************
2249 template< typename MT1, typename MT2 >
2250 struct RowExprTrait< SMatDMatMultExpr<MT1,MT2> >
2251 {
2252  public:
2253  //**********************************************************************************************
2254  typedef typename MultExprTrait< typename RowExprTrait<const MT1>::Type, MT2 >::Type Type;
2255  //**********************************************************************************************
2256 };
2258 //*************************************************************************************************
2259 
2260 
2261 //*************************************************************************************************
2263 template< typename MT1, typename MT2 >
2264 struct ColumnExprTrait< SMatDMatMultExpr<MT1,MT2> >
2265 {
2266  public:
2267  //**********************************************************************************************
2268  typedef typename MultExprTrait< MT1, typename ColumnExprTrait<const MT2>::Type >::Type Type;
2269  //**********************************************************************************************
2270 };
2272 //*************************************************************************************************
2273 
2274 } // namespace blaze
2275 
2276 #endif
const MT::ElementType max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1649
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:89
LeftOperand lhs_
Left-hand side sparse matrix of the multiplication expression.
Definition: SMatDMatMultExpr.h:432
Header file for the SMatDVecMultExprTrait class template.
Header file for the Rows type trait.
Header file for the IsUniUpper type trait.
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:8247
IntrinsicTrait< ElementType >::Type IntrinsicType
Resulting intrinsic element type.
Definition: SMatDMatMultExpr.h:230
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:105
Header file for basic type definitions.
SelectType< evaluateRight, const RT2, CT2 >::Type RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: SMatDMatMultExpr.h:244
BLAZE_ALWAYS_INLINE size_t size(const Vector< VT, TF > &vector)
Returns the current size/dimension of the vector.
Definition: Vector.h:264
bool isAligned() const
Returns whether the operands of the expression are properly aligned in memory.
Definition: SMatDMatMultExpr.h:415
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:258
Header file for the IsSparseMatrix type trait.
Efficient implementation of a compressed matrix.The CompressedMatrix class template is the represent...
Definition: CompressedMatrix.h:209
Header file for the IsDiagonal type trait.
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:79
SelectType< IsExpression< MT1 >::value, const MT1, const MT1 & >::Type LeftOperand
Composite type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:235
Header file for the ColumnExprTrait class template.
Header file for the IsSame and IsStrictlySame type traits.
SMatDMatMultExpr(const MT1 &lhs, const MT2 &rhs)
Constructor for the SMatDMatMultExpr class.
Definition: SMatDMatMultExpr.h:266
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:821
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:2507
Header file for the IsRowVector type trait.
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:261
Header file for the And class template.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:90
Header file for the TDVecSMatMultExprTrait class template.
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:699
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Expression object for sparse matrix-dense matrix multiplications.The SMatDMatMultExpr class represent...
Definition: Forward.h:89
Type relationship analysis.This class tests if the two data types A and B are equal. For this type comparison, the cv-qualifiers of both data types are ignored. If A and B are the same data type (ignoring the cv-qualifiers), then the value member enumeration is set to 1, the nested type definition Type is TrueType, and the class derives from TrueType. Otherwise value is set to 0, Type is FalseType, and the class derives from FalseType.
Definition: IsSame.h:158
ResultType::TransposeType TransposeType
Transpose type for expression template evaluations.
Definition: SMatDMatMultExpr.h:228
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:90
Header file for the RequiresEvaluation type trait.
Header file for the TSVecSMatMultExprTrait class template.
SelectType< evaluateLeft, const RT1, CT1 >::Type LT
Type for the assignment of the left-hand side sparse matrix operand.
Definition: SMatDMatMultExpr.h:241
Header file for the IsUniLower type trait.
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:107
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:861
Constraint on the data type.
Constraint on the data type.
Header file for the MultExprTrait class template.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:90
RT1::ElementType ET1
Element type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:129
Compile time type selection.The SelectType class template selects one of the two given types T1 and T...
Definition: SelectType.h:59
Header file for the DisableIf class template.
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Header file for the IsSymmetric type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: StorageOrder.h:161
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: SMatDMatMultExpr.h:281
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2511
Header file for the Or class template.
BLAZE_ALWAYS_INLINE EnableIf< And< IsIntegral< T >, HasSize< T, 2UL > > >::Type store(T *address, const sse_int16_t &value)
Aligned store of a vector of 2-byte integral values.
Definition: Store.h:80
const MT::ElementType min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1602
Header file for the DenseMatrix base class.
BLAZE_ALWAYS_INLINE void assign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the assignment of a matrix to a matrix.
Definition: Matrix.h:635
Header file for the Columns type trait.
Header file for the DMatDVecMultExprTrait class template.
Header file for the IsLower type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:92
ResultType::ElementType ElementType
Resulting element type.
Definition: SMatDMatMultExpr.h:229
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:78
Header file for the IsTriangular type trait.
bool isAliased(const T *alias) const
Returns whether the expression is aliased with the given address alias.
Definition: SMatDMatMultExpr.h:405
Constraints on the storage order of matrix types.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
SelectType< IsExpression< MT2 >::value, const MT2, const MT2 & >::Type RightOperand
Composite type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:238
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:2505
Header file for the SelectType class template.
Header file for the RowExprTrait class template.
Header file for all forward declarations for expression class templates.
Header file for the IsDenseMatrix type trait.
MultTrait< RT1, RT2 >::Type ResultType
Result type for expression template evaluations.
Definition: SMatDMatMultExpr.h:226
MT1::CompositeType CT1
Composite type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:131
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
const size_t SMP_SMATDMATMULT_THRESHOLD
SMP row-major sparse matrix/row-major dense matrix multiplication threshold.This threshold specifies ...
Definition: Thresholds.h:1018
Header file for the serial shim.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:165
LeftOperand leftOperand() const
Returns the left-hand side sparse matrix operand.
Definition: SMatDMatMultExpr.h:371
EnableIf< IsDenseMatrix< MT1 > >::Type smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
Header file for the IsSparseVector type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type, a compilation error is created.
Definition: Symmetric.h:116
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: StorageOrder.h:81
const ElementType ReturnType
Return type for expression template evaluations.
Definition: SMatDMatMultExpr.h:231
Removal of reference modifiers.The RemoveCV type trait removes any reference modifiers from the given...
Definition: RemoveReference.h:69
Intrinsic characteristics of data types.The IntrinsicTrait class template provides the intrinsic char...
Definition: IntrinsicTrait.h:749
MT2::CompositeType CT2
Composite type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:132
Header file for run time assertion macros.
EnableIf< IsDenseMatrix< MT1 > >::Type smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Utility type for generic codes.
Base template for the MultTrait class.
Definition: MultTrait.h:150
size_t rows() const
Returns the current number of rows of the matrix.
Definition: SMatDMatMultExpr.h:351
BLAZE_ALWAYS_INLINE void addAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the addition assignment of a matrix to a matrix.
Definition: Matrix.h:742
Substitution Failure Is Not An Error (SFINAE) class.The EnableIf class template is an auxiliary tool ...
Definition: EnableIf.h:184
Header file for the reset shim.
SMatDMatMultExpr< MT1, MT2 > This
Type of this SMatDMatMultExpr instance.
Definition: SMatDMatMultExpr.h:225
bool canSMPAssign() const
Returns whether the expression can be used in SMP assignments.
Definition: SMatDMatMultExpr.h:425
MT2::ResultType RT2
Result type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:128
RightOperand rhs_
Right-hand side dense matrix of the multiplication expression.
Definition: SMatDMatMultExpr.h:433
Header file for the RemoveReference type trait.
Substitution Failure Is Not An Error (SFINAE) class.The DisableIf class template is an auxiliary tool...
Definition: DisableIf.h:184
BLAZE_ALWAYS_INLINE EnableIf< And< IsIntegral< T >, HasSize< T, 2UL > >, sse_int16_t >::Type set(T value)
Sets all values in the vector to the given 2-byte integral value.
Definition: Set.h:73
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:283
Header file for the IsDenseVector type trait.
Header file for all intrinsic functionality.
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
MT1::ResultType RT1
Result type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:127
Header file for the IsRowMajorMatrix type trait.
const DMatTransExpr< MT,!SO > trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:937
Header file for the IsComputation type trait class.
Header file for the TDVecDMatMultExprTrait class template.
EnableIf< IsDenseMatrix< MT1 > >::Type smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
This ResultType
Result type for expression template evaluations.
Definition: CompressedMatrix.h:2502
Header file for the IsTrue value trait.
Header file for the TSVecDMatMultExprTrait class template.
bool canAlias(const T *alias) const
Returns whether the expression can alias with the given address alias.
Definition: SMatDMatMultExpr.h:393
const ResultType CompositeType
Data type for composite expression templates.
Definition: SMatDMatMultExpr.h:232
Header file for the IsUpper type trait.
Header file for the DMatSVecMultExprTrait class template.
Header file for the IsColumnVector type trait.
ResultType::OppositeType OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: SMatDMatMultExpr.h:227
RightOperand rightOperand() const
Returns the right-hand side dense matrix operand.
Definition: SMatDMatMultExpr.h:381
Constraint on the data type.
RT2::ElementType ET2
Element type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:130
Header file for the IsResizable type trait.
size_t columns() const
Returns the current number of columns of the matrix.
Definition: SMatDMatMultExpr.h:361
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:79
Header file for the IsExpression type trait class.
Header file for the FunctionTrace class.
BLAZE_ALWAYS_INLINE void subAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the subtraction assignment of a matrix to matrix.
Definition: Matrix.h:849