SMatDMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
53 #include <blaze/math/Functions.h>
54 #include <blaze/math/Intrinsics.h>
55 #include <blaze/math/shims/Reset.h>
93 #include <blaze/util/Assert.h>
94 #include <blaze/util/EnableIf.h>
95 #include <blaze/util/Exception.h>
96 #include <blaze/util/InvalidType.h>
98 #include <blaze/util/mpl/And.h>
99 #include <blaze/util/mpl/Or.h>
100 #include <blaze/util/SelectType.h>
101 #include <blaze/util/Types.h>
105 
106 
107 namespace blaze {
108 
109 //=================================================================================================
110 //
111 // CLASS SMATDMATMULTEXPR
112 //
113 //=================================================================================================
114 
115 //*************************************************************************************************
122 template< typename MT1 // Type of the left-hand side sparse matrix
123  , typename MT2 > // Type of the right-hand side dense matrix
124 class SMatDMatMultExpr : public DenseMatrix< SMatDMatMultExpr<MT1,MT2>, false >
125  , private MatMatMultExpr
126  , private Computation
127 {
128  private:
129  //**Type definitions****************************************************************************
130  typedef typename MT1::ResultType RT1;
131  typedef typename MT2::ResultType RT2;
132  typedef typename RT1::ElementType ET1;
133  typedef typename RT2::ElementType ET2;
134  typedef typename MT1::CompositeType CT1;
135  typedef typename MT2::CompositeType CT2;
136  //**********************************************************************************************
137 
138  //**********************************************************************************************
141  //**********************************************************************************************
142 
143  //**********************************************************************************************
145  enum { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
146  //**********************************************************************************************
147 
148  //**********************************************************************************************
150 
154  template< typename T1, typename T2, typename T3 >
155  struct IsEvaluationRequired {
156  enum { value = ( evaluateLeft || evaluateRight ) };
157  };
159  //**********************************************************************************************
160 
161  //**********************************************************************************************
163 
166  template< typename T1, typename T2, typename T3 >
167  struct UseVectorizedKernel {
168  enum { value = useOptimizedKernels &&
169  !IsDiagonal<T3>::value &&
170  T1::vectorizable && T3::vectorizable &&
171  IsRowMajorMatrix<T1>::value &&
172  IsSame<typename T1::ElementType,typename T2::ElementType>::value &&
173  IsSame<typename T1::ElementType,typename T3::ElementType>::value &&
174  IntrinsicTrait<typename T1::ElementType>::addition &&
175  IntrinsicTrait<typename T1::ElementType>::subtraction &&
176  IntrinsicTrait<typename T1::ElementType>::multiplication };
177  };
179  //**********************************************************************************************
180 
181  //**********************************************************************************************
183 
187  template< typename T1, typename T2, typename T3 >
188  struct UseOptimizedKernel {
189  enum { value = useOptimizedKernels &&
190  !UseVectorizedKernel<T1,T2,T3>::value &&
191  !IsDiagonal<T3>::value &&
192  !IsResizable<typename T1::ElementType>::value &&
193  !IsResizable<ET1>::value };
194  };
196  //**********************************************************************************************
197 
198  //**********************************************************************************************
200 
203  template< typename T1, typename T2, typename T3 >
204  struct UseDefaultKernel {
205  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
206  !UseOptimizedKernel<T1,T2,T3>::value };
207  };
209  //**********************************************************************************************
210 
211  public:
212  //**Type definitions****************************************************************************
219  typedef const ElementType ReturnType;
220  typedef const ResultType CompositeType;
221 
223  typedef typename SelectType< IsExpression<MT1>::value, const MT1, const MT1& >::Type LeftOperand;
224 
226  typedef typename SelectType< IsExpression<MT2>::value, const MT2, const MT2& >::Type RightOperand;
227 
230 
233  //**********************************************************************************************
234 
235  //**Compilation flags***************************************************************************
237  enum { vectorizable = !IsDiagonal<MT2>::value &&
238  MT2::vectorizable &&
242 
244  enum { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
245  !evaluateRight && MT2::smpAssignable };
246  //**********************************************************************************************
247 
248  //**Constructor*********************************************************************************
254  explicit inline SMatDMatMultExpr( const MT1& lhs, const MT2& rhs )
255  : lhs_( lhs ) // Left-hand side sparse matrix of the multiplication expression
256  , rhs_( rhs ) // Right-hand side dense matrix of the multiplication expression
257  {
258  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
259  }
260  //**********************************************************************************************
261 
262  //**Access operator*****************************************************************************
269  inline ReturnType operator()( size_t i, size_t j ) const {
270  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
271  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
272 
274 
275  ElementType tmp = ElementType();
276 
277  // Early exit
278  if( lhs_.columns() == 0UL )
279  return tmp;
280 
281  // Fast computation in case the left-hand side sparse matrix directly provides iterators
283  {
284  CT1 A( lhs_ ); // Evaluation of the left-hand side sparse matrix operand
285 
286  const ConstIterator end( ( IsUpper<MT2>::value )
287  ?( IsStrictlyUpper<MT2>::value ? A.lowerBound(i,j) : A.upperBound(i,j) )
288  :( A.end(i) ) );
289  ConstIterator element( ( IsLower<MT2>::value )
290  ?( IsStrictlyLower<MT2>::value ? A.upperBound(i,j) : A.lowerBound(i,j) )
291  :( A.begin(i) ) );
292 
293  if( element != end ) {
294  tmp = element->value() * rhs_(element->index(),j);
295  ++element;
296  for( ; element!=end; ++element ) {
297  tmp += element->value() * rhs_(element->index(),j);
298  }
299  }
300  }
301 
302  // Default computation in case the left-hand side sparse matrix doesn't provide iterators
303  else
304  {
305  const size_t kbegin( ( IsUpper<MT1>::value )
306  ?( ( IsLower<MT2>::value )
307  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
308  , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
309  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
310  :( ( IsLower<MT2>::value )
311  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
312  :( 0UL ) ) );
313  const size_t kend( ( IsLower<MT1>::value )
314  ?( ( IsUpper<MT2>::value )
315  ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
316  , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
317  :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
318  :( ( IsUpper<MT2>::value )
319  ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
320  :( lhs_.columns() ) ) );
321 
322  if( ( !IsTriangular<MT1>::value && !IsTriangular<MT2>::value ) || kbegin < kend ) {
323  tmp = lhs_(i,kbegin) * rhs_(kbegin,j);
324  for( size_t k=kbegin+1UL; k<kend; ++k ) {
325  tmp += lhs_(i,k) * rhs_(k,j);
326  }
327  }
328  }
329 
330  return tmp;
331  }
332  //**********************************************************************************************
333 
334  //**At function*********************************************************************************
342  inline ReturnType at( size_t i, size_t j ) const {
343  if( i >= lhs_.rows() ) {
344  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
345  }
346  if( j >= rhs_.columns() ) {
347  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
348  }
349  return (*this)(i,j);
350  }
351  //**********************************************************************************************
352 
353  //**Rows function*******************************************************************************
358  inline size_t rows() const {
359  return lhs_.rows();
360  }
361  //**********************************************************************************************
362 
363  //**Columns function****************************************************************************
368  inline size_t columns() const {
369  return rhs_.columns();
370  }
371  //**********************************************************************************************
372 
373  //**Left operand access*************************************************************************
378  inline LeftOperand leftOperand() const {
379  return lhs_;
380  }
381  //**********************************************************************************************
382 
383  //**Right operand access************************************************************************
388  inline RightOperand rightOperand() const {
389  return rhs_;
390  }
391  //**********************************************************************************************
392 
393  //**********************************************************************************************
399  template< typename T >
400  inline bool canAlias( const T* alias ) const {
401  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
402  }
403  //**********************************************************************************************
404 
405  //**********************************************************************************************
411  template< typename T >
412  inline bool isAliased( const T* alias ) const {
413  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
414  }
415  //**********************************************************************************************
416 
417  //**********************************************************************************************
422  inline bool isAligned() const {
423  return rhs_.isAligned();
424  }
425  //**********************************************************************************************
426 
427  //**********************************************************************************************
432  inline bool canSMPAssign() const {
433  return ( rows() > SMP_SMATDMATMULT_THRESHOLD );
434  }
435  //**********************************************************************************************
436 
437  private:
438  //**Member variables****************************************************************************
439  LeftOperand lhs_;
440  RightOperand rhs_;
441  //**********************************************************************************************
442 
443  //**Assignment to dense matrices****************************************************************
456  template< typename MT // Type of the target dense matrix
457  , bool SO > // Storage order of the target dense matrix
458  friend inline void assign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
459  {
461 
462  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
463  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
464 
465  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
466  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
467 
468  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
469  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
470  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
471  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
472  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
473  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
474 
475  SMatDMatMultExpr::selectAssignKernel( ~lhs, A, B );
476  }
478  //**********************************************************************************************
479 
480  //**Default assignment to dense matrices********************************************************
494  template< typename MT3 // Type of the left-hand side target matrix
495  , typename MT4 // Type of the left-hand side matrix operand
496  , typename MT5 > // Type of the right-hand side matrix operand
497  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
498  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
499  {
500  typedef typename MT4::ConstIterator ConstIterator;
501 
502  const size_t block( Or< IsRowMajorMatrix<MT3>, IsDiagonal<MT5> >::value ? B.columns() : 64UL );
503 
504  reset( C );
505 
506  for( size_t jj=0UL; jj<B.columns(); jj+=block )
507  {
508  const size_t jtmp( min( jj+block, B.columns() ) );
509 
510  for( size_t i=0UL; i<A.rows(); ++i )
511  {
512  ConstIterator element( A.begin(i) );
513  const ConstIterator end( A.end(i) );
514 
515  for( ; element!=end; ++element )
516  {
517  const size_t i1( element->index() );
518 
520  {
521  C(i,i1) = element->value() * B(i1,i1);
522  }
523  else
524  {
525  const size_t jbegin( ( IsUpper<MT5>::value )
526  ?( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
527  :( jj ) );
528  const size_t jend( ( IsLower<MT5>::value )
529  ?( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
530  :( jtmp ) );
531 
532  if( IsTriangular<MT5>::value && jbegin >= jend )
533  continue;
534 
535  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
536 
537  for( size_t j=jbegin; j<jend; ++j ) {
538  if( isDefault( C(i,j) ) )
539  C(i,j) = element->value() * B(i1,j);
540  else
541  C(i,j) += element->value() * B(i1,j);
542  }
543  }
544  }
545  }
546  }
547  }
549  //**********************************************************************************************
550 
551  //**Optimized assignment to dense matrices******************************************************
565  template< typename MT3 // Type of the left-hand side target matrix
566  , typename MT4 // Type of the left-hand side matrix operand
567  , typename MT5 > // Type of the right-hand side matrix operand
568  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
569  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
570  {
571  typedef typename MT4::ConstIterator ConstIterator;
572 
573  const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 64UL );
574 
575  reset( C );
576 
577  for( size_t jj=0UL; jj<B.columns(); jj+=block )
578  {
579  const size_t jtmp( min( jj+block, B.columns() ) );
580 
581  for( size_t i=0UL; i<A.rows(); ++i )
582  {
583  const ConstIterator end( A.end(i) );
584  ConstIterator element( A.begin(i) );
585 
586  const size_t nonzeros( A.nonZeros(i) );
587  const size_t kpos( nonzeros & size_t(-4) );
588  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
589 
590  for( size_t k=0UL; k<kpos; k+=4UL )
591  {
592  const size_t i1( element->index() );
593  const ET1 v1( element->value() );
594  ++element;
595  const size_t i2( element->index() );
596  const ET1 v2( element->value() );
597  ++element;
598  const size_t i3( element->index() );
599  const ET1 v3( element->value() );
600  ++element;
601  const size_t i4( element->index() );
602  const ET1 v4( element->value() );
603  ++element;
604 
605  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
606 
607  const size_t jbegin( ( IsUpper<MT5>::value )
608  ?( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
609  :( jj ) );
610  const size_t jend( ( IsLower<MT5>::value )
611  ?( min( jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) )
612  :( jtmp ) );
613 
614  if( IsTriangular<MT5>::value && jbegin >= jend )
615  continue;
616 
617  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
618 
619  const size_t jnum( jend - jbegin );
620  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
621  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
622 
623  for( size_t j=jbegin; j<jpos; j+=4UL ) {
624  C(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
625  C(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
626  C(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
627  C(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
628  }
629  for( size_t j=jpos; j<jend; ++j ) {
630  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
631  }
632  }
633 
634  for( ; element!=end; ++element )
635  {
636  const size_t i1( element->index() );
637  const ET1 v1( element->value() );
638 
639  const size_t jbegin( ( IsUpper<MT5>::value )
640  ?( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
641  :( jj ) );
642  const size_t jend( ( IsLower<MT5>::value )
643  ?( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
644  :( jtmp ) );
645 
646  if( IsTriangular<MT5>::value && jbegin >= jend )
647  continue;
648 
649  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
650 
651  const size_t jnum( jend - jbegin );
652  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
653  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
654 
655  for( size_t j=jbegin; j<jpos; j+=4UL ) {
656  C(i,j ) += v1 * B(i1,j );
657  C(i,j+1UL) += v1 * B(i1,j+1UL);
658  C(i,j+2UL) += v1 * B(i1,j+2UL);
659  C(i,j+3UL) += v1 * B(i1,j+3UL);
660  }
661  for( size_t j=jpos; j<jend; ++j ) {
662  C(i,j) += v1 * B(i1,j);
663  }
664  }
665  }
666  }
667  }
669  //**********************************************************************************************
670 
671  //**Vectorized assignment to dense matrices*****************************************************
685  template< typename MT3 // Type of the left-hand side target matrix
686  , typename MT4 // Type of the left-hand side matrix operand
687  , typename MT5 > // Type of the right-hand side matrix operand
688  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
689  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
690  {
691  typedef IntrinsicTrait<ElementType> IT;
692  typedef typename MT4::ConstIterator ConstIterator;
693 
694  const bool remainder( !IsPadded<MT3>::value || !IsPadded<MT5>::value );
695 
696  reset( C );
697 
698  for( size_t i=0UL; i<A.rows(); ++i )
699  {
700  const ConstIterator end( A.end(i) );
701  ConstIterator element( A.begin(i) );
702 
703  const size_t nonzeros( A.nonZeros(i) );
704  const size_t kpos( nonzeros & size_t(-4) );
705  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
706 
707  for( size_t k=0UL; k<kpos; k+=4UL )
708  {
709  const size_t i1( element->index() );
710  const ET1 v1( element->value() );
711  ++element;
712  const size_t i2( element->index() );
713  const ET1 v2( element->value() );
714  ++element;
715  const size_t i3( element->index() );
716  const ET1 v3( element->value() );
717  ++element;
718  const size_t i4( element->index() );
719  const ET1 v4( element->value() );
720  ++element;
721 
722  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
723 
724  const IntrinsicType xmm1( set( v1 ) );
725  const IntrinsicType xmm2( set( v2 ) );
726  const IntrinsicType xmm3( set( v3 ) );
727  const IntrinsicType xmm4( set( v4 ) );
728 
729  const size_t jbegin( ( IsUpper<MT5>::value )
730  ?( ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
731  :( 0UL ) );
732  const size_t jend( ( IsLower<MT5>::value )
733  ?( IsStrictlyLower<MT5>::value ? i4 : i4+1UL )
734  :( B.columns() ) );
735  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
736 
737  const size_t jpos( remainder ? ( jend & size_t(-IT::size) ) : jend );
738  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (IT::size) ) ) == jpos, "Invalid end calculation" );
739 
740  size_t j( jbegin );
741 
742  for( ; j<jpos; j+=IT::size ) {
743  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) + xmm2 * B.load(i2,j) + xmm3 * B.load(i3,j) + xmm4 * B.load(i4,j) );
744  }
745  for( ; remainder && j<jend; ++j ) {
746  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
747  }
748  }
749 
750  for( ; element!=end; ++element )
751  {
752  const size_t i1( element->index() );
753  const ET1 v1( element->value() );
754 
755  const IntrinsicType xmm1( set( v1 ) );
756 
757  const size_t jbegin( ( IsUpper<MT5>::value )
758  ?( ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
759  :( 0UL ) );
760  const size_t jend( ( IsLower<MT5>::value )
761  ?( IsStrictlyLower<MT5>::value ? i1 : i1+1UL )
762  :( B.columns() ) );
763  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
764 
765  const size_t jpos( remainder ? ( jend & size_t(-IT::size) ) : jend );
766  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (IT::size) ) ) == jpos, "Invalid end calculation" );
767 
768  size_t j( jbegin );
769 
770  for( ; j<jpos; j+=IT::size ) {
771  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) );
772  }
773  for( ; remainder && j<jend; ++j ) {
774  C(i,j) += v1 * B(i1,j);
775  }
776  }
777  }
778  }
780  //**********************************************************************************************
781 
782  //**Assignment to sparse matrices***************************************************************
795  template< typename MT // Type of the target sparse matrix
796  , bool SO > // Storage order of the target sparse matrix
797  friend inline void assign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
798  {
800 
801  typedef typename SelectType< SO, OppositeType, ResultType >::Type TmpType;
802 
809 
810  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
811  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
812 
813  const TmpType tmp( serial( rhs ) );
814  assign( ~lhs, tmp );
815  }
817  //**********************************************************************************************
818 
819  //**Addition assignment to dense matrices*******************************************************
832  template< typename MT // Type of the target dense matrix
833  , bool SO > // Storage order of the target dense matrix
834  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
835  {
837 
838  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
839  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
840 
841  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
842  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
843 
844  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
845  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
846  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
847  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
848  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
849  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
850 
851  SMatDMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
852  }
854  //**********************************************************************************************
855 
856  //**Default addition assignment to dense matrices***********************************************
870  template< typename MT3 // Type of the left-hand side target matrix
871  , typename MT4 // Type of the left-hand side matrix operand
872  , typename MT5 > // Type of the right-hand side matrix operand
873  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
874  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
875  {
876  typedef typename MT4::ConstIterator ConstIterator;
877 
878  const size_t block( Or< IsRowMajorMatrix<MT3>, IsDiagonal<MT5> >::value ? B.columns() : 64UL );
879 
880  for( size_t jj=0UL; jj<B.columns(); jj+=block )
881  {
882  const size_t jtmp( min( jj+block, B.columns() ) );
883 
884  for( size_t i=0UL; i<A.rows(); ++i )
885  {
886  const ConstIterator end( A.end(i) );
887  ConstIterator element( A.begin(i) );
888 
889  for( ; element!=end; ++element )
890  {
891  const size_t i1( element->index() );
892 
893  if( IsDiagonal<MT5>::value )
894  {
895  C(i,i1) += element->value() * B(i1,i1);
896  }
897  else
898  {
899  const size_t jbegin( ( IsUpper<MT5>::value )
900  ?( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
901  :( jj ) );
902  const size_t jend( ( IsLower<MT5>::value )
903  ?( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
904  :( jtmp ) );
905 
906  if( IsTriangular<MT5>::value && jbegin >= jend )
907  continue;
908 
909  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
910 
911  const size_t jnum( jend - jbegin );
912  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
913  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
914 
915  for( size_t j=jbegin; j<jpos; j+=4UL ) {
916  C(i,j ) += element->value() * B(i1,j );
917  C(i,j+1UL) += element->value() * B(i1,j+1UL);
918  C(i,j+2UL) += element->value() * B(i1,j+2UL);
919  C(i,j+3UL) += element->value() * B(i1,j+3UL);
920  }
921  for( size_t j=jpos; j<jend; ++j ) {
922  C(i,j) += element->value() * B(i1,j);
923  }
924  }
925  }
926  }
927  }
928  }
930  //**********************************************************************************************
931 
932  //**Optimized addition assignment to dense matrices*********************************************
946  template< typename MT3 // Type of the left-hand side target matrix
947  , typename MT4 // Type of the left-hand side matrix operand
948  , typename MT5 > // Type of the right-hand side matrix operand
949  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
950  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
951  {
952  typedef typename MT4::ConstIterator ConstIterator;
953 
954  const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 64UL );
955 
956  for( size_t jj=0UL; jj<B.columns(); jj+=block )
957  {
958  const size_t jtmp( min( jj+block, B.columns() ) );
959 
960  for( size_t i=0UL; i<A.rows(); ++i )
961  {
962  const ConstIterator end( A.end(i) );
963  ConstIterator element( A.begin(i) );
964 
965  const size_t nonzeros( A.nonZeros(i) );
966  const size_t kpos( nonzeros & size_t(-4) );
967  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
968 
969  for( size_t k=0UL; k<kpos; k+=4UL )
970  {
971  const size_t i1( element->index() );
972  const ET1 v1( element->value() );
973  ++element;
974  const size_t i2( element->index() );
975  const ET1 v2( element->value() );
976  ++element;
977  const size_t i3( element->index() );
978  const ET1 v3( element->value() );
979  ++element;
980  const size_t i4( element->index() );
981  const ET1 v4( element->value() );
982  ++element;
983 
984  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
985 
986  const size_t jbegin( ( IsUpper<MT5>::value )
987  ?( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
988  :( jj ) );
989  const size_t jend( ( IsLower<MT5>::value )
990  ?( min( jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) )
991  :( jtmp ) );
992 
993  if( IsTriangular<MT5>::value && jbegin >= jend )
994  continue;
995 
996  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
997 
998  const size_t jnum( jend - jbegin );
999  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1000  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1001 
1002  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1003  C(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1004  C(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1005  C(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1006  C(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1007  }
1008  for( size_t j=jpos; j<jend; ++j ) {
1009  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1010  }
1011  }
1012 
1013  for( ; element!=end; ++element )
1014  {
1015  const size_t i1( element->index() );
1016  const ET1 v1( element->value() );
1017 
1018  const size_t jbegin( ( IsUpper<MT5>::value )
1019  ?( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1020  :( jj ) );
1021  const size_t jend( ( IsLower<MT5>::value )
1022  ?( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
1023  :( jtmp ) );
1024 
1025  if( IsTriangular<MT5>::value && jbegin >= jend )
1026  continue;
1027 
1028  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1029 
1030  const size_t jnum( jend - jbegin );
1031  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1032  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1033 
1034  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1035  C(i,j ) += v1 * B(i1,j );
1036  C(i,j+1UL) += v1 * B(i1,j+1UL);
1037  C(i,j+2UL) += v1 * B(i1,j+2UL);
1038  C(i,j+3UL) += v1 * B(i1,j+3UL);
1039  }
1040  for( size_t j=jpos; j<jend; ++j ) {
1041  C(i,j) += v1 * B(i1,j);
1042  }
1043  }
1044  }
1045  }
1046  }
1048  //**********************************************************************************************
1049 
1050  //**Vectorized addition assignment to dense matrices********************************************
1064  template< typename MT3 // Type of the left-hand side target matrix
1065  , typename MT4 // Type of the left-hand side matrix operand
1066  , typename MT5 > // Type of the right-hand side matrix operand
1067  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
1068  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1069  {
1070  typedef IntrinsicTrait<ElementType> IT;
1071  typedef typename MT4::ConstIterator ConstIterator;
1072 
1073  const bool remainder( !IsPadded<MT3>::value || !IsPadded<MT5>::value );
1074 
1075  for( size_t i=0UL; i<A.rows(); ++i )
1076  {
1077  const ConstIterator end( A.end(i) );
1078  ConstIterator element( A.begin(i) );
1079 
1080  const size_t nonzeros( A.nonZeros(i) );
1081  const size_t kpos( nonzeros & size_t(-4) );
1082  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1083 
1084  for( size_t k=0UL; k<kpos; k+=4UL )
1085  {
1086  const size_t i1( element->index() );
1087  const ET1 v1( element->value() );
1088  ++element;
1089  const size_t i2( element->index() );
1090  const ET1 v2( element->value() );
1091  ++element;
1092  const size_t i3( element->index() );
1093  const ET1 v3( element->value() );
1094  ++element;
1095  const size_t i4( element->index() );
1096  const ET1 v4( element->value() );
1097  ++element;
1098 
1099  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1100 
1101  const IntrinsicType xmm1( set( v1 ) );
1102  const IntrinsicType xmm2( set( v2 ) );
1103  const IntrinsicType xmm3( set( v3 ) );
1104  const IntrinsicType xmm4( set( v4 ) );
1105 
1106  const size_t jbegin( ( IsUpper<MT5>::value )
1107  ?( ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
1108  :( 0UL ) );
1109  const size_t jend( ( IsLower<MT5>::value )
1110  ?( IsStrictlyLower<MT5>::value ? i4 : i4+1UL )
1111  :( B.columns() ) );
1112  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1113 
1114  const size_t jpos( remainder ? ( jend & size_t(-IT::size) ) : jend );
1115  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (IT::size) ) ) == jpos, "Invalid end calculation" );
1116 
1117  size_t j( jbegin );
1118 
1119  for( ; j<jpos; j+=IT::size ) {
1120  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) + xmm2 * B.load(i2,j) + xmm3 * B.load(i3,j) + xmm4 * B.load(i4,j) );
1121  }
1122  for( ; remainder && j<jend; ++j ) {
1123  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1124  }
1125  }
1126 
1127  for( ; element!=end; ++element )
1128  {
1129  const size_t i1( element->index() );
1130  const ET1 v1( element->value() );
1131 
1132  const IntrinsicType xmm1( set( v1 ) );
1133 
1134  const size_t jbegin( ( IsUpper<MT5>::value )
1135  ?( ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
1136  :( 0UL ) );
1137  const size_t jend( ( IsLower<MT5>::value )
1138  ?( IsStrictlyLower<MT5>::value ? i1 : i1+1UL )
1139  :( B.columns() ) );
1140  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1141 
1142  const size_t jpos( remainder ? ( jend & size_t(-IT::size) ) : jend );
1143  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (IT::size) ) ) == jpos, "Invalid end calculation" );
1144 
1145  size_t j( jbegin );
1146 
1147  for( ; j<jpos; j+=IT::size ) {
1148  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) );
1149  }
1150  for( ; remainder && j<jend; ++j ) {
1151  C(i,j) += v1 * B(i1,j);
1152  }
1153  }
1154  }
1155  }
1157  //**********************************************************************************************
1158 
1159  //**Addition assignment to sparse matrices******************************************************
1160  // No special implementation for the addition assignment to sparse matrices.
1161  //**********************************************************************************************
1162 
1163  //**Subtraction assignment to dense matrices****************************************************
1176  template< typename MT // Type of the target dense matrix
1177  , bool SO > // Storage order of the target dense matrix
1178  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1179  {
1181 
1182  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1183  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1184 
1185  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
1186  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
1187 
1188  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1189  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1190  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1191  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1192  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1193  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1194 
1195  SMatDMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1196  }
1198  //**********************************************************************************************
1199 
1200  //**Default subtraction assignment to dense matrices********************************************
1214  template< typename MT3 // Type of the left-hand side target matrix
1215  , typename MT4 // Type of the left-hand side matrix operand
1216  , typename MT5 > // Type of the right-hand side matrix operand
1217  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
1218  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1219  {
1220  typedef typename MT4::ConstIterator ConstIterator;
1221 
1222  const size_t block( Or< IsRowMajorMatrix<MT3>, IsDiagonal<MT5> >::value ? B.columns() : 64UL );
1223 
1224  for( size_t jj=0UL; jj<B.columns(); jj+=block )
1225  {
1226  const size_t jtmp( min( jj+block, B.columns() ) );
1227 
1228  for( size_t i=0UL; i<A.rows(); ++i )
1229  {
1230  const ConstIterator end( A.end(i) );
1231  ConstIterator element( A.begin(i) );
1232 
1233  for( ; element!=end; ++element )
1234  {
1235  const size_t i1( element->index() );
1236 
1237  if( IsDiagonal<MT5>::value )
1238  {
1239  C(i,i1) -= element->value() * B(i1,i1);
1240  }
1241  else
1242  {
1243  const size_t jbegin( ( IsUpper<MT5>::value )
1244  ?( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1245  :( jj ) );
1246  const size_t jend( ( IsLower<MT5>::value )
1247  ?( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
1248  :( jtmp ) );
1249 
1250  if( IsTriangular<MT5>::value && jbegin >= jend )
1251  continue;
1252 
1253  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1254 
1255  const size_t jnum( jend - jbegin );
1256  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1257  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1258 
1259  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1260  C(i,j ) -= element->value() * B(i1,j );
1261  C(i,j+1UL) -= element->value() * B(i1,j+1UL);
1262  C(i,j+2UL) -= element->value() * B(i1,j+2UL);
1263  C(i,j+3UL) -= element->value() * B(i1,j+3UL);
1264  }
1265  for( size_t j=jpos; j<jend; ++j ) {
1266  C(i,j) -= element->value() * B(i1,j);
1267  }
1268  }
1269  }
1270  }
1271  }
1272  }
1274  //**********************************************************************************************
1275 
1276  //**Optimized subtraction assignment to dense matrices******************************************
1290  template< typename MT3 // Type of the left-hand side target matrix
1291  , typename MT4 // Type of the left-hand side matrix operand
1292  , typename MT5 > // Type of the right-hand side matrix operand
1293  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
1294  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1295  {
1296  typedef typename MT4::ConstIterator ConstIterator;
1297 
1298  const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 64UL );
1299 
1300  for( size_t jj=0UL; jj<B.columns(); jj+=block )
1301  {
1302  const size_t jtmp( min( jj+block, B.columns() ) );
1303 
1304  for( size_t i=0UL; i<A.rows(); ++i )
1305  {
1306  const ConstIterator end( A.end(i) );
1307  ConstIterator element( A.begin(i) );
1308 
1309  const size_t nonzeros( A.nonZeros(i) );
1310  const size_t kpos( nonzeros & size_t(-4) );
1311  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1312 
1313  for( size_t k=0UL; k<kpos; k+=4UL )
1314  {
1315  const size_t i1( element->index() );
1316  const ET1 v1( element->value() );
1317  ++element;
1318  const size_t i2( element->index() );
1319  const ET1 v2( element->value() );
1320  ++element;
1321  const size_t i3( element->index() );
1322  const ET1 v3( element->value() );
1323  ++element;
1324  const size_t i4( element->index() );
1325  const ET1 v4( element->value() );
1326  ++element;
1327 
1328  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1329 
1330  const size_t jbegin( ( IsUpper<MT5>::value )
1331  ?( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1332  :( jj ) );
1333  const size_t jend( ( IsLower<MT5>::value )
1334  ?( min( jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) )
1335  :( jtmp ) );
1336 
1337  if( IsTriangular<MT5>::value && jbegin >= jend )
1338  continue;
1339 
1340  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1341 
1342  const size_t jnum( jend - jbegin );
1343  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1344  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1345 
1346  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1347  C(i,j ) -= v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1348  C(i,j+1UL) -= v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1349  C(i,j+2UL) -= v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1350  C(i,j+3UL) -= v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1351  }
1352  for( size_t j=jpos; j<jend; ++j ) {
1353  C(i,j) -= v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1354  }
1355  }
1356 
1357  for( ; element!=end; ++element )
1358  {
1359  const size_t i1( element->index() );
1360  const ET1 v1( element->value() );
1361 
1362  const size_t jbegin( ( IsUpper<MT5>::value )
1363  ?( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1364  :( jj ) );
1365  const size_t jend( ( IsLower<MT5>::value )
1366  ?( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
1367  :( jtmp ) );
1368 
1369  if( IsTriangular<MT5>::value && jbegin >= jend )
1370  continue;
1371 
1372  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1373 
1374  const size_t jnum( jend - jbegin );
1375  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1376  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1377 
1378  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1379  C(i,j ) -= v1 * B(i1,j );
1380  C(i,j+1UL) -= v1 * B(i1,j+1UL);
1381  C(i,j+2UL) -= v1 * B(i1,j+2UL);
1382  C(i,j+3UL) -= v1 * B(i1,j+3UL);
1383  }
1384  for( size_t j=jpos; j<jend; ++j ) {
1385  C(i,j) -= v1 * B(i1,j);
1386  }
1387  }
1388  }
1389  }
1390  }
1392  //**********************************************************************************************
1393 
1394  //**Vectorized subtraction assignment to dense matrices*****************************************
1408  template< typename MT3 // Type of the left-hand side target matrix
1409  , typename MT4 // Type of the left-hand side matrix operand
1410  , typename MT5 > // Type of the right-hand side matrix operand
1411  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
1412  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1413  {
1414  typedef IntrinsicTrait<ElementType> IT;
1415  typedef typename MT4::ConstIterator ConstIterator;
1416 
1417  const bool remainder( !IsPadded<MT3>::value || !IsPadded<MT5>::value );
1418 
1419  for( size_t i=0UL; i<A.rows(); ++i )
1420  {
1421  const ConstIterator end( A.end(i) );
1422  ConstIterator element( A.begin(i) );
1423 
1424  const size_t nonzeros( A.nonZeros(i) );
1425  const size_t kpos( nonzeros & size_t(-4) );
1426  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1427 
1428  for( size_t k=0UL; k<kpos; k+=4UL )
1429  {
1430  const size_t i1( element->index() );
1431  const ET1 v1( element->value() );
1432  ++element;
1433  const size_t i2( element->index() );
1434  const ET1 v2( element->value() );
1435  ++element;
1436  const size_t i3( element->index() );
1437  const ET1 v3( element->value() );
1438  ++element;
1439  const size_t i4( element->index() );
1440  const ET1 v4( element->value() );
1441  ++element;
1442 
1443  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1444 
1445  const IntrinsicType xmm1( set( v1 ) );
1446  const IntrinsicType xmm2( set( v2 ) );
1447  const IntrinsicType xmm3( set( v3 ) );
1448  const IntrinsicType xmm4( set( v4 ) );
1449 
1450  const size_t jbegin( ( IsUpper<MT5>::value )
1451  ?( ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
1452  :( 0UL ) );
1453  const size_t jend( ( IsLower<MT5>::value )
1454  ?( IsStrictlyLower<MT5>::value ? i4 : i4+1UL )
1455  :( B.columns() ) );
1456  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1457 
1458  const size_t jpos( remainder ? ( jend & size_t(-IT::size) ) : jend );
1459  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (IT::size) ) ) == jpos, "Invalid end calculation" );
1460 
1461  size_t j( jbegin );
1462 
1463  for( ; j<jpos; j+=IT::size ) {
1464  C.store( i, j, C.load(i,j) - xmm1 * B.load(i1,j) - xmm2 * B.load(i2,j) - xmm3 * B.load(i3,j) - xmm4 * B.load(i4,j) );
1465  }
1466  for( ; remainder && j<jend; ++j ) {
1467  C(i,j) -= v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1468  }
1469  }
1470 
1471  for( ; element!=end; ++element )
1472  {
1473  const size_t i1( element->index() );
1474  const ET1 v1( element->value() );
1475 
1476  const IntrinsicType xmm1( set( v1 ) );
1477 
1478  const size_t jbegin( ( IsUpper<MT5>::value )
1479  ?( ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
1480  :( 0UL ) );
1481  const size_t jend( ( IsLower<MT5>::value )
1482  ?( IsStrictlyLower<MT5>::value ? i1 : i1+1UL )
1483  :( B.columns() ) );
1484  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1485 
1486  const size_t jpos( remainder ? ( jend & size_t(-IT::size) ) : jend );
1487  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (IT::size) ) ) == jpos, "Invalid end calculation" );
1488 
1489  size_t j( jbegin );
1490 
1491  for( ; j<jpos; j+=IT::size ) {
1492  C.store( i, j, C.load(i,j) - xmm1 * B.load(i1,j) );
1493  }
1494  for( ; remainder && j<jend; ++j ) {
1495  C(i,j) -= v1 * B(i1,j);
1496  }
1497  }
1498  }
1499  }
1501  //**********************************************************************************************
1502 
1503  //**Subtraction assignment to sparse matrices***************************************************
1504  // No special implementation for the subtraction assignment to sparse matrices.
1505  //**********************************************************************************************
1506 
1507  //**Multiplication assignment to dense matrices*************************************************
1508  // No special implementation for the multiplication assignment to dense matrices.
1509  //**********************************************************************************************
1510 
1511  //**Multiplication assignment to sparse matrices************************************************
1512  // No special implementation for the multiplication assignment to sparse matrices.
1513  //**********************************************************************************************
1514 
1515  //**SMP assignment to dense matrices************************************************************
1530  template< typename MT // Type of the target dense matrix
1531  , bool SO > // Storage order of the target dense matrix
1532  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1533  smpAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1534  {
1536 
1537  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1538  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1539 
1540  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1541  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1542 
1543  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1544  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1545  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1546  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1547  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1548  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1549 
1550  smpAssign( ~lhs, A * B );
1551  }
1553  //**********************************************************************************************
1554 
1555  //**SMP assignment to sparse matrices***********************************************************
1570  template< typename MT // Type of the target sparse matrix
1571  , bool SO > // Storage order of the target sparse matrix
1572  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1573  smpAssign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1574  {
1576 
1577  typedef typename SelectType< SO, OppositeType, ResultType >::Type TmpType;
1578 
1585 
1586  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1587  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1588 
1589  const TmpType tmp( rhs );
1590  smpAssign( ~lhs, tmp );
1591  }
1593  //**********************************************************************************************
1594 
1595  //**SMP addition assignment to dense matrices***************************************************
1611  template< typename MT // Type of the target dense matrix
1612  , bool SO > // Storage order of the target dense matrix
1613  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1614  smpAddAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1615  {
1617 
1618  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1619  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1620 
1621  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1622  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1623 
1624  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1625  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1626  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1627  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1628  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1629  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1630 
1631  smpAddAssign( ~lhs, A * B );
1632  }
1634  //**********************************************************************************************
1635 
1636  //**SMP addition assignment to sparse matrices**************************************************
1637  // No special implementation for the SMP addition assignment to sparse matrices.
1638  //**********************************************************************************************
1639 
1640  //**SMP subtraction assignment to dense matrices************************************************
1656  template< typename MT // Type of the target dense matrix
1657  , bool SO > // Storage order of the target dense matrix
1658  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1659  smpSubAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1660  {
1662 
1663  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1664  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1665 
1666  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1667  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1668 
1669  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1670  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1671  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1672  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1673  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1674  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1675 
1676  smpSubAssign( ~lhs, A * B );
1677  }
1679  //**********************************************************************************************
1680 
1681  //**SMP subtraction assignment to sparse matrices***********************************************
1682  // No special implementation for the SMP subtraction assignment to sparse matrices.
1683  //**********************************************************************************************
1684 
1685  //**SMP multiplication assignment to dense matrices*********************************************
1686  // No special implementation for the SMP multiplication assignment to dense matrices.
1687  //**********************************************************************************************
1688 
1689  //**SMP multiplication assignment to sparse matrices********************************************
1690  // No special implementation for the SMP multiplication assignment to sparse matrices.
1691  //**********************************************************************************************
1692 
1693  //**Compile time checks*************************************************************************
1701  //**********************************************************************************************
1702 };
1703 //*************************************************************************************************
1704 
1705 
1706 
1707 
1708 //=================================================================================================
1709 //
1710 // GLOBAL BINARY ARITHMETIC OPERATORS
1711 //
1712 //=================================================================================================
1713 
1714 //*************************************************************************************************
1743 template< typename T1 // Type of the left-hand side sparse matrix
1744  , typename T2 > // Type of the right-hand side dense matrix
1745 inline const SMatDMatMultExpr<T1,T2>
1747 {
1749 
1750  if( (~lhs).columns() != (~rhs).rows() ) {
1751  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1752  }
1753 
1754  return SMatDMatMultExpr<T1,T2>( ~lhs, ~rhs );
1755 }
1756 //*************************************************************************************************
1757 
1758 
1759 
1760 
1761 //=================================================================================================
1762 //
1763 // ROWS SPECIALIZATIONS
1764 //
1765 //=================================================================================================
1766 
1767 //*************************************************************************************************
1769 template< typename MT1, typename MT2 >
1770 struct Rows< SMatDMatMultExpr<MT1,MT2> > : public Rows<MT1>
1771 {};
1773 //*************************************************************************************************
1774 
1775 
1776 
1777 
1778 //=================================================================================================
1779 //
1780 // COLUMNS SPECIALIZATIONS
1781 //
1782 //=================================================================================================
1783 
1784 //*************************************************************************************************
1786 template< typename MT1, typename MT2 >
1787 struct Columns< SMatDMatMultExpr<MT1,MT2> > : public Columns<MT2>
1788 {};
1790 //*************************************************************************************************
1791 
1792 
1793 
1794 
1795 //=================================================================================================
1796 //
1797 // ISALIGNED SPECIALIZATIONS
1798 //
1799 //=================================================================================================
1800 
1801 //*************************************************************************************************
1803 template< typename MT1, typename MT2 >
1804 struct IsAligned< SMatDMatMultExpr<MT1,MT2> > : public IsTrue< IsAligned<MT2>::value >
1805 {};
1807 //*************************************************************************************************
1808 
1809 
1810 
1811 
1812 //=================================================================================================
1813 //
1814 // ISLOWER SPECIALIZATIONS
1815 //
1816 //=================================================================================================
1817 
1818 //*************************************************************************************************
1820 template< typename MT1, typename MT2 >
1821 struct IsLower< SMatDMatMultExpr<MT1,MT2> >
1822  : public IsTrue< And< IsLower<MT1>, IsLower<MT2> >::value >
1823 {};
1825 //*************************************************************************************************
1826 
1827 
1828 
1829 
1830 //=================================================================================================
1831 //
1832 // ISUNILOWER SPECIALIZATIONS
1833 //
1834 //=================================================================================================
1835 
1836 //*************************************************************************************************
1838 template< typename MT1, typename MT2 >
1839 struct IsUniLower< SMatDMatMultExpr<MT1,MT2> >
1840  : public IsTrue< And< IsUniLower<MT1>, IsUniLower<MT2> >::value >
1841 {};
1843 //*************************************************************************************************
1844 
1845 
1846 
1847 
1848 //=================================================================================================
1849 //
1850 // ISSTRICTLYLOWER SPECIALIZATIONS
1851 //
1852 //=================================================================================================
1853 
1854 //*************************************************************************************************
1856 template< typename MT1, typename MT2 >
1857 struct IsStrictlyLower< SMatDMatMultExpr<MT1,MT2> >
1858  : public IsTrue< Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
1859  , And< IsStrictlyLower<MT2>, IsLower<MT1> > >::value >
1860 {};
1862 //*************************************************************************************************
1863 
1864 
1865 
1866 
1867 //=================================================================================================
1868 //
1869 // ISUPPER SPECIALIZATIONS
1870 //
1871 //=================================================================================================
1872 
1873 //*************************************************************************************************
1875 template< typename MT1, typename MT2 >
1876 struct IsUpper< SMatDMatMultExpr<MT1,MT2> >
1877  : public IsTrue< And< IsUpper<MT1>, IsUpper<MT2> >::value >
1878 {};
1880 //*************************************************************************************************
1881 
1882 
1883 
1884 
1885 //=================================================================================================
1886 //
1887 // ISUNIUPPER SPECIALIZATIONS
1888 //
1889 //=================================================================================================
1890 
1891 //*************************************************************************************************
1893 template< typename MT1, typename MT2 >
1894 struct IsUniUpper< SMatDMatMultExpr<MT1,MT2> >
1895  : public IsTrue< And< IsUniUpper<MT1>, IsUniUpper<MT2> >::value >
1896 {};
1898 //*************************************************************************************************
1899 
1900 
1901 
1902 
1903 //=================================================================================================
1904 //
1905 // ISSTRICTLYUPPER SPECIALIZATIONS
1906 //
1907 //=================================================================================================
1908 
1909 //*************************************************************************************************
1911 template< typename MT1, typename MT2 >
1912 struct IsStrictlyUpper< SMatDMatMultExpr<MT1,MT2> >
1913  : public IsTrue< Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
1914  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > >::value >
1915 {};
1917 //*************************************************************************************************
1918 
1919 
1920 
1921 
1922 //=================================================================================================
1923 //
1924 // EXPRESSION TRAIT SPECIALIZATIONS
1925 //
1926 //=================================================================================================
1927 
1928 //*************************************************************************************************
1930 template< typename MT1, typename MT2, typename VT >
1931 struct DMatDVecMultExprTrait< SMatDMatMultExpr<MT1,MT2>, VT >
1932 {
1933  public:
1934  //**********************************************************************************************
1935  typedef typename SelectType< IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
1936  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value &&
1937  IsDenseVector<VT>::value && IsColumnVector<VT>::value
1938  , typename SMatDVecMultExprTrait< MT1, typename DMatDVecMultExprTrait<MT2,VT>::Type >::Type
1939  , INVALID_TYPE >::Type Type;
1940  //**********************************************************************************************
1941 };
1943 //*************************************************************************************************
1944 
1945 
1946 //*************************************************************************************************
1948 template< typename MT1, typename MT2, typename VT >
1949 struct DMatSVecMultExprTrait< SMatDMatMultExpr<MT1,MT2>, VT >
1950 {
1951  public:
1952  //**********************************************************************************************
1953  typedef typename SelectType< IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
1954  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value &&
1955  IsSparseVector<VT>::value && IsColumnVector<VT>::value
1956  , typename SMatDVecMultExprTrait< MT1, typename DMatSVecMultExprTrait<MT2,VT>::Type >::Type
1957  , INVALID_TYPE >::Type Type;
1958  //**********************************************************************************************
1959 };
1961 //*************************************************************************************************
1962 
1963 
1964 //*************************************************************************************************
1966 template< typename VT, typename MT1, typename MT2 >
1967 struct TDVecDMatMultExprTrait< VT, SMatDMatMultExpr<MT1,MT2> >
1968 {
1969  public:
1970  //**********************************************************************************************
1971  typedef typename SelectType< IsDenseVector<VT>::value && IsRowVector<VT>::value &&
1972  IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
1973  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value
1974  , typename TDVecDMatMultExprTrait< typename TDVecSMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
1975  , INVALID_TYPE >::Type Type;
1976  //**********************************************************************************************
1977 };
1979 //*************************************************************************************************
1980 
1981 
1982 //*************************************************************************************************
1984 template< typename VT, typename MT1, typename MT2 >
1985 struct TSVecDMatMultExprTrait< VT, SMatDMatMultExpr<MT1,MT2> >
1986 {
1987  public:
1988  //**********************************************************************************************
1989  typedef typename SelectType< IsSparseVector<VT>::value && IsRowVector<VT>::value &&
1990  IsSparseMatrix<MT1>::value && IsRowMajorMatrix<MT1>::value &&
1991  IsDenseMatrix<MT2>::value && IsRowMajorMatrix<MT2>::value
1992  , typename TSVecDMatMultExprTrait< typename TSVecSMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
1993  , INVALID_TYPE >::Type Type;
1994  //**********************************************************************************************
1995 };
1997 //*************************************************************************************************
1998 
1999 
2000 //*************************************************************************************************
2002 template< typename MT1, typename MT2, bool AF >
2003 struct SubmatrixExprTrait< SMatDMatMultExpr<MT1,MT2>, AF >
2004 {
2005  public:
2006  //**********************************************************************************************
2007  typedef typename MultExprTrait< typename SubmatrixExprTrait<const MT1,AF>::Type
2008  , typename SubmatrixExprTrait<const MT2,AF>::Type >::Type Type;
2009  //**********************************************************************************************
2010 };
2012 //*************************************************************************************************
2013 
2014 
2015 //*************************************************************************************************
2017 template< typename MT1, typename MT2 >
2018 struct RowExprTrait< SMatDMatMultExpr<MT1,MT2> >
2019 {
2020  public:
2021  //**********************************************************************************************
2022  typedef typename MultExprTrait< typename RowExprTrait<const MT1>::Type, MT2 >::Type Type;
2023  //**********************************************************************************************
2024 };
2026 //*************************************************************************************************
2027 
2028 
2029 //*************************************************************************************************
2031 template< typename MT1, typename MT2 >
2032 struct ColumnExprTrait< SMatDMatMultExpr<MT1,MT2> >
2033 {
2034  public:
2035  //**********************************************************************************************
2036  typedef typename MultExprTrait< MT1, typename ColumnExprTrait<const MT2>::Type >::Type Type;
2037  //**********************************************************************************************
2038 };
2040 //*************************************************************************************************
2041 
2042 } // namespace blaze
2043 
2044 #endif
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exceptionThis macro encapsulates the default way of...
Definition: Exception.h:187
const MT::ElementType max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1729
BLAZE_ALWAYS_INLINE EnableIf< And< IsIntegral< T >, HasSize< T, 2UL > >, simd_int16_t >::Type set(T value)
Sets all values in the vector to the given 2-byte integral value.
Definition: Set.h:73
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:89
Header file for mathematical functions.
LeftOperand lhs_
Left-hand side sparse matrix of the multiplication expression.
Definition: SMatDMatMultExpr.h:439
Header file for the SMatDVecMultExprTrait class template.
Header file for the Rows type trait.
Header file for the IsUniUpper type trait.
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:7820
IntrinsicTrait< ElementType >::Type IntrinsicType
Resulting intrinsic element type.
Definition: SMatDMatMultExpr.h:218
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:105
Header file for basic type definitions.
SelectType< evaluateRight, const RT2, CT2 >::Type RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: SMatDMatMultExpr.h:232
BLAZE_ALWAYS_INLINE size_t size(const Vector< VT, TF > &vector)
Returns the current size/dimension of the vector.
Definition: Vector.h:252
bool isAligned() const
Returns whether the operands of the expression are properly aligned in memory.
Definition: SMatDMatMultExpr.h:422
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:250
Header file for the IsSparseMatrix type trait.
Efficient implementation of a compressed matrix.The CompressedMatrix class template is the represent...
Definition: CompressedMatrix.h:207
Header file for the IsDiagonal type trait.
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:79
SelectType< IsExpression< MT1 >::value, const MT1, const MT1 & >::Type LeftOperand
Composite type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:223
Header file for the ColumnExprTrait class template.
Header file for the IsSame and IsStrictlySame type traits.
SMatDMatMultExpr(const MT1 &lhs, const MT2 &rhs)
Constructor for the SMatDMatMultExpr class.
Definition: SMatDMatMultExpr.h:254
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:507
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:2588
Header file for the IsRowVector type trait.
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:259
Header file for the And class template.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:90
Header file for the TDVecSMatMultExprTrait class template.
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:721
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Expression object for sparse matrix-dense matrix multiplications.The SMatDMatMultExpr class represent...
Definition: Forward.h:97
Type relationship analysis.This class tests if the two data types A and B are equal. For this type comparison, the cv-qualifiers of both data types are ignored. If A and B are the same data type (ignoring the cv-qualifiers), then the value member enumeration is set to 1, the nested type definition Type is TrueType, and the class derives from TrueType. Otherwise value is set to 0, Type is FalseType, and the class derives from FalseType.
Definition: IsSame.h:158
ResultType::TransposeType TransposeType
Transpose type for expression template evaluations.
Definition: SMatDMatMultExpr.h:216
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:90
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Header file for the TSVecSMatMultExprTrait class template.
SelectType< evaluateLeft, const RT1, CT1 >::Type LT
Type for the assignment of the left-hand side sparse matrix operand.
Definition: SMatDMatMultExpr.h:229
Header file for the IsUniLower type trait.
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:117
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:547
Constraint on the data type.
Constraint on the data type.
Header file for the MultExprTrait class template.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:90
RT1::ElementType ET1
Element type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:132
Compile time type selection.The SelectType class template selects one of the two given types T1 and T...
Definition: SelectType.h:59
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
Compile time check for row-major matrix types.This type trait tests whether or not the given template...
Definition: IsRowMajorMatrix.h:110
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:79
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: SMatDMatMultExpr.h:269
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2592
Header file for the Or class template.
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exceptionThis macro encapsulates the default way of Bla...
Definition: Exception.h:331
const MT::ElementType min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1682
Header file for the DenseMatrix base class.
Header file for the Columns type trait.
Header file for the DMatDVecMultExprTrait class template.
Header file for the IsLower type trait.
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:92
ResultType::ElementType ElementType
Resulting element type.
Definition: SMatDMatMultExpr.h:217
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:78
Header file for the IsTriangular type trait.
bool isAliased(const T *alias) const
Returns whether the expression is aliased with the given address alias.
Definition: SMatDMatMultExpr.h:412
Constraints on the storage order of matrix types.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
SelectType< IsExpression< MT2 >::value, const MT2, const MT2 & >::Type RightOperand
Composite type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:226
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:2586
Header file for the SelectType class template.
Header file for the RowExprTrait class template.
Header file for all forward declarations for expression class templates.
Header file for the IsDenseMatrix type trait.
MultTrait< RT1, RT2 >::Type ResultType
Result type for expression template evaluations.
Definition: SMatDMatMultExpr.h:214
MT1::CompositeType CT1
Composite type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:134
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
Header file for the serial shim.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:165
LeftOperand leftOperand() const
Returns the left-hand side sparse matrix operand.
Definition: SMatDMatMultExpr.h:378
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: SMatDMatMultExpr.h:342
EnableIf< IsDenseMatrix< MT1 > >::Type smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
Header file for the IsSparseVector type trait.
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:79
const ElementType ReturnType
Return type for expression template evaluations.
Definition: SMatDMatMultExpr.h:219
Removal of reference modifiers.The RemoveCV type trait removes any reference modifiers from the given...
Definition: RemoveReference.h:69
Intrinsic characteristics of data types.The IntrinsicTrait class template provides the intrinsic char...
Definition: IntrinsicTrait.h:1232
MT2::CompositeType CT2
Composite type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:135
Header file for run time assertion macros.
EnableIf< IsDenseMatrix< MT1 > >::Type smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Utility type for generic codes.
Base template for the MultTrait class.
Definition: MultTrait.h:138
size_t rows() const
Returns the current number of rows of the matrix.
Definition: SMatDMatMultExpr.h:358
Substitution Failure Is Not An Error (SFINAE) class.The EnableIf class template is an auxiliary tool ...
Definition: EnableIf.h:184
const bool useOptimizedKernels
Configuration switch for optimized kernels.This configuration switch enables/disables all optimized c...
Definition: Optimizations.h:84
Header file for the reset shim.
SMatDMatMultExpr< MT1, MT2 > This
Type of this SMatDMatMultExpr instance.
Definition: SMatDMatMultExpr.h:213
bool canSMPAssign() const
Returns whether the expression can be used in SMP assignments.
Definition: SMatDMatMultExpr.h:432
Constraints on the storage order of matrix types.
MT2::ResultType RT2
Result type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:131
RightOperand rhs_
Right-hand side dense matrix of the multiplication expression.
Definition: SMatDMatMultExpr.h:440
Header file for the RemoveReference type trait.
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:122
Header file for the IsDenseVector type trait.
Header file for all intrinsic functionality.
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
MT1::ResultType RT1
Result type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:130
Header file for the IsRowMajorMatrix type trait.
Header file for the IsComputation type trait class.
Header file for the TDVecDMatMultExprTrait class template.
Compile time logical or evaluation.The Or class template performs at compile time a logical or ('&&')...
Definition: Or.h:78
EnableIf< IsDenseMatrix< MT1 > >::Type smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
This ResultType
Result type for expression template evaluations.
Definition: CompressedMatrix.h:2583
Header file for the IsTrue value trait.
Header file for the TSVecDMatMultExprTrait class template.
bool canAlias(const T *alias) const
Returns whether the expression can alias with the given address alias.
Definition: SMatDMatMultExpr.h:400
const ResultType CompositeType
Data type for composite expression templates.
Definition: SMatDMatMultExpr.h:220
Header file for the IsUpper type trait.
Header file for exception macros.
Header file for the DMatSVecMultExprTrait class template.
Header file for the IsColumnVector type trait.
ResultType::OppositeType OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: SMatDMatMultExpr.h:215
RightOperand rightOperand() const
Returns the right-hand side dense matrix operand.
Definition: SMatDMatMultExpr.h:388
Constraint on the data type.
RT2::ElementType ET2
Element type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:133
Header file for the IsResizable type trait.
size_t columns() const
Returns the current number of columns of the matrix.
Definition: SMatDMatMultExpr.h:368
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:79
Header file for the IsExpression type trait class.
Header file for the FunctionTrace class.