SMatDMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
50 #include <blaze/math/Exception.h>
55 #include <blaze/math/Functions.h>
63 #include <blaze/math/shims/Reset.h>
65 #include <blaze/math/SIMD.h>
109 #include <blaze/system/Thresholds.h>
110 #include <blaze/util/Assert.h>
112 #include <blaze/util/EnableIf.h>
115 #include <blaze/util/InvalidType.h>
116 #include <blaze/util/mpl/And.h>
117 #include <blaze/util/mpl/Bool.h>
118 #include <blaze/util/mpl/If.h>
119 #include <blaze/util/mpl/Or.h>
120 #include <blaze/util/Types.h>
123 
124 
125 namespace blaze {
126 
127 //=================================================================================================
128 //
129 // CLASS SMATDMATMULTEXPR
130 //
131 //=================================================================================================
132 
133 //*************************************************************************************************
140 template< typename MT1 // Type of the left-hand side sparse matrix
141  , typename MT2 // Type of the right-hand side dense matrix
142  , bool SF // Symmetry flag
143  , bool HF // Hermitian flag
144  , bool LF // Lower flag
145  , bool UF > // Upper flag
146 class SMatDMatMultExpr : public DenseMatrix< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, false >
147  , private MatMatMultExpr
148  , private Computation
149 {
150  private:
151  //**Type definitions****************************************************************************
158  //**********************************************************************************************
159 
160  //**********************************************************************************************
162  enum : bool { evaluateLeft = IsComputation<MT1>::value || RequiresEvaluation<MT1>::value };
163  //**********************************************************************************************
164 
165  //**********************************************************************************************
167  enum : bool { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
168  //**********************************************************************************************
169 
170  //**********************************************************************************************
172  enum : bool {
173  SYM = ( SF && !( HF || LF || UF ) ),
174  HERM = ( HF && !( LF || UF ) ),
175  LOW = ( LF || ( ( SF || HF ) && UF ) ),
176  UPP = ( UF || ( ( SF || HF ) && LF ) )
177  };
178  //**********************************************************************************************
179 
180  //**********************************************************************************************
182 
186  template< typename T1, typename T2, typename T3 >
187  struct IsEvaluationRequired {
188  enum : bool { value = ( evaluateLeft || evaluateRight ) };
189  };
191  //**********************************************************************************************
192 
193  //**********************************************************************************************
195 
198  template< typename T1, typename T2, typename T3 >
199  struct UseVectorizedKernel {
200  enum : bool { value = useOptimizedKernels &&
202  T1::simdEnabled && T3::simdEnabled &&
206  , ElementType_<T3> >::value &&
209  };
211  //**********************************************************************************************
212 
213  //**********************************************************************************************
215 
219  template< typename T1, typename T2, typename T3 >
220  struct UseOptimizedKernel {
221  enum : bool { value = useOptimizedKernels &&
222  !UseVectorizedKernel<T1,T2,T3>::value &&
224  !IsResizable< ElementType_<T1> >::value &&
226  };
228  //**********************************************************************************************
229 
230  //**********************************************************************************************
232 
235  template< typename T1, typename T2, typename T3 >
236  struct UseDefaultKernel {
237  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
238  !UseOptimizedKernel<T1,T2,T3>::value };
239  };
241  //**********************************************************************************************
242 
243  //**********************************************************************************************
245 
248  typedef IfTrue_< HERM
249  , DeclHerm
250  , IfTrue_< SYM
251  , DeclSym
252  , IfTrue_< LOW
253  , IfTrue_< UPP
254  , DeclDiag
255  , DeclLow >
256  , IfTrue_< UPP
257  , DeclUpp
258  , Noop > > > > ForwardFunctor;
260  //**********************************************************************************************
261 
262  public:
263  //**Type definitions****************************************************************************
266 
272  typedef const ElementType ReturnType;
273  typedef const ResultType CompositeType;
274 
276  typedef If_< IsExpression<MT1>, const MT1, const MT1& > LeftOperand;
277 
279  typedef If_< IsExpression<MT2>, const MT2, const MT2& > RightOperand;
280 
283 
286  //**********************************************************************************************
287 
288  //**Compilation flags***************************************************************************
290  enum : bool { simdEnabled = !IsDiagonal<MT2>::value &&
291  MT2::simdEnabled &&
294 
296  enum : bool { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
297  !evaluateRight && MT2::smpAssignable };
298  //**********************************************************************************************
299 
300  //**SIMD properties*****************************************************************************
302  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
303  //**********************************************************************************************
304 
305  //**Constructor*********************************************************************************
311  explicit inline SMatDMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
312  : lhs_( lhs ) // Left-hand side sparse matrix of the multiplication expression
313  , rhs_( rhs ) // Right-hand side dense matrix of the multiplication expression
314  {
315  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
316  }
317  //**********************************************************************************************
318 
319  //**Access operator*****************************************************************************
326  inline ReturnType operator()( size_t i, size_t j ) const {
327  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
328  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
329 
330  if( IsDiagonal<MT1>::value ) {
331  return lhs_(i,i) * rhs_(i,j);
332  }
333  else if( IsDiagonal<MT2>::value ) {
334  return lhs_(i,j) * rhs_(j,j);
335  }
337  const size_t begin( ( IsUpper<MT1>::value )
338  ?( ( IsLower<MT2>::value )
339  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
340  , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
341  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
342  :( ( IsLower<MT2>::value )
343  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
344  :( 0UL ) ) );
345  const size_t end( ( IsLower<MT1>::value )
346  ?( ( IsUpper<MT2>::value )
347  ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
348  , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
349  :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
350  :( ( IsUpper<MT2>::value )
351  ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
352  :( lhs_.columns() ) ) );
353 
354  if( begin >= end ) return ElementType();
355 
356  const size_t n( end - begin );
357 
358  return subvector( row( lhs_, i ), begin, n ) * subvector( column( rhs_, j ), begin, n );
359  }
360  else {
361  return row( lhs_, i ) * column( rhs_, j );
362  }
363  }
364  //**********************************************************************************************
365 
366  //**At function*********************************************************************************
374  inline ReturnType at( size_t i, size_t j ) const {
375  if( i >= lhs_.rows() ) {
376  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
377  }
378  if( j >= rhs_.columns() ) {
379  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
380  }
381  return (*this)(i,j);
382  }
383  //**********************************************************************************************
384 
385  //**Rows function*******************************************************************************
390  inline size_t rows() const noexcept {
391  return lhs_.rows();
392  }
393  //**********************************************************************************************
394 
395  //**Columns function****************************************************************************
400  inline size_t columns() const noexcept {
401  return rhs_.columns();
402  }
403  //**********************************************************************************************
404 
405  //**Left operand access*************************************************************************
410  inline LeftOperand leftOperand() const noexcept {
411  return lhs_;
412  }
413  //**********************************************************************************************
414 
415  //**Right operand access************************************************************************
420  inline RightOperand rightOperand() const noexcept {
421  return rhs_;
422  }
423  //**********************************************************************************************
424 
425  //**********************************************************************************************
431  template< typename T >
432  inline bool canAlias( const T* alias ) const noexcept {
433  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
434  }
435  //**********************************************************************************************
436 
437  //**********************************************************************************************
443  template< typename T >
444  inline bool isAliased( const T* alias ) const noexcept {
445  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
446  }
447  //**********************************************************************************************
448 
449  //**********************************************************************************************
454  inline bool isAligned() const noexcept {
455  return rhs_.isAligned();
456  }
457  //**********************************************************************************************
458 
459  //**********************************************************************************************
464  inline bool canSMPAssign() const noexcept {
465  return ( rows() * columns() >= SMP_SMATDMATMULT_THRESHOLD ) && !IsDiagonal<MT2>::value;
466  }
467  //**********************************************************************************************
468 
469  private:
470  //**Member variables****************************************************************************
471  LeftOperand lhs_;
472  RightOperand rhs_;
473  //**********************************************************************************************
474 
475  //**Assignment to dense matrices****************************************************************
488  template< typename MT // Type of the target dense matrix
489  , bool SO > // Storage order of the target dense matrix
490  friend inline void assign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
491  {
493 
494  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
495  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
496 
497  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
498  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
499 
500  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
501  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
502  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
503  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
504  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
505  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
506 
507  SMatDMatMultExpr::selectAssignKernel( ~lhs, A, B );
508  }
510  //**********************************************************************************************
511 
512  //**Default assignment to dense matrices********************************************************
526  template< typename MT3 // Type of the left-hand side target matrix
527  , typename MT4 // Type of the left-hand side matrix operand
528  , typename MT5 > // Type of the right-hand side matrix operand
530  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
531  {
533 
534  const size_t block( Or< IsRowMajorMatrix<MT3>, IsDiagonal<MT5> >::value ? B.columns() : 64UL );
535 
536  reset( C );
537 
538  for( size_t jj=0UL; jj<B.columns(); jj+=block )
539  {
540  const size_t jtmp( min( jj+block, B.columns() ) );
541 
542  for( size_t i=0UL; i<A.rows(); ++i )
543  {
544  ConstIterator element( A.begin(i) );
545  const ConstIterator end( A.end(i) );
546 
547  for( ; element!=end; ++element )
548  {
549  const size_t i1( element->index() );
550 
552  {
553  C(i,i1) = element->value() * B(i1,i1);
554  }
555  else
556  {
557  const size_t jbegin( ( IsUpper<MT5>::value )
558  ?( ( UPP )
559  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
560  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
561  :( jj ) );
562  const size_t jend( ( IsLower<MT5>::value )
563  ?( ( SYM || HERM || LOW )
564  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
565  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
566  :( SYM || HERM || LOW ? min(i+1UL,jtmp) : jtmp ) );
567 
568  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
569  continue;
570 
571  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
572 
573  for( size_t j=jbegin; j<jend; ++j ) {
574  if( isDefault( C(i,j) ) )
575  C(i,j) = element->value() * B(i1,j);
576  else
577  C(i,j) += element->value() * B(i1,j);
578  }
579  }
580  }
581  }
582  }
583 
584  if( SYM || HERM ) {
585  for( size_t i=0UL; i<A.rows(); ++i ) {
586  for( size_t j=i+1UL; j<B.columns(); ++j ) {
587  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
588  }
589  }
590  }
591  }
593  //**********************************************************************************************
594 
595  //**Optimized assignment to dense matrices******************************************************
609  template< typename MT3 // Type of the left-hand side target matrix
610  , typename MT4 // Type of the left-hand side matrix operand
611  , typename MT5 > // Type of the right-hand side matrix operand
613  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
614  {
616 
617  const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 64UL );
618 
619  reset( C );
620 
621  for( size_t jj=0UL; jj<B.columns(); jj+=block )
622  {
623  const size_t jtmp( min( jj+block, B.columns() ) );
624 
625  for( size_t i=0UL; i<A.rows(); ++i )
626  {
627  const ConstIterator end( A.end(i) );
628  ConstIterator element( A.begin(i) );
629 
630  const size_t nonzeros( A.nonZeros(i) );
631  const size_t kpos( nonzeros & size_t(-4) );
632  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
633 
634  for( size_t k=0UL; k<kpos; k+=4UL )
635  {
636  const size_t i1( element->index() );
637  const ET1 v1( element->value() );
638  ++element;
639  const size_t i2( element->index() );
640  const ET1 v2( element->value() );
641  ++element;
642  const size_t i3( element->index() );
643  const ET1 v3( element->value() );
644  ++element;
645  const size_t i4( element->index() );
646  const ET1 v4( element->value() );
647  ++element;
648 
649  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
650 
651  const size_t jbegin( ( IsUpper<MT5>::value )
652  ?( ( UPP )
653  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
654  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
655  :( UPP ? max(i,jj) : jj ) );
656  const size_t jend( ( IsLower<MT5>::value )
657  ?( ( SYM || HERM || LOW )
658  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) )
659  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) ) )
660  :( SYM || HERM || LOW ? min(i+1UL,jtmp) : jtmp ) );
661 
662  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
663  continue;
664 
665  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
666 
667  const size_t jnum( jend - jbegin );
668  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
669  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
670 
671  for( size_t j=jbegin; j<jpos; j+=4UL ) {
672  C(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
673  C(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
674  C(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
675  C(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
676  }
677  for( size_t j=jpos; j<jend; ++j ) {
678  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
679  }
680  }
681 
682  for( ; element!=end; ++element )
683  {
684  const size_t i1( element->index() );
685  const ET1 v1( element->value() );
686 
687  const size_t jbegin( ( IsUpper<MT5>::value )
688  ?( ( UPP )
689  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
690  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
691  :( UPP ? max(i,jj) : jj ) );
692  const size_t jend( ( IsLower<MT5>::value )
693  ?( ( SYM || HERM || LOW )
694  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
695  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
696  :( SYM || HERM || LOW ? min(i+1UL,jtmp) : jtmp ) );
697 
698  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
699  continue;
700 
701  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
702 
703  const size_t jnum( jend - jbegin );
704  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
705  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
706 
707  for( size_t j=jbegin; j<jpos; j+=4UL ) {
708  C(i,j ) += v1 * B(i1,j );
709  C(i,j+1UL) += v1 * B(i1,j+1UL);
710  C(i,j+2UL) += v1 * B(i1,j+2UL);
711  C(i,j+3UL) += v1 * B(i1,j+3UL);
712  }
713  for( size_t j=jpos; j<jend; ++j ) {
714  C(i,j) += v1 * B(i1,j);
715  }
716  }
717  }
718  }
719 
720  if( SYM || HERM ) {
721  for( size_t i=0UL; i<A.rows(); ++i ) {
722  for( size_t j=i+1UL; j<B.columns(); ++j ) {
723  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
724  }
725  }
726  }
727  }
729  //**********************************************************************************************
730 
731  //**Vectorized assignment to dense matrices*****************************************************
745  template< typename MT3 // Type of the left-hand side target matrix
746  , typename MT4 // Type of the left-hand side matrix operand
747  , typename MT5 > // Type of the right-hand side matrix operand
749  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
750  {
752 
753  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT5>::value );
754 
755  reset( C );
756 
757  for( size_t i=0UL; i<A.rows(); ++i )
758  {
759  const ConstIterator end( A.end(i) );
760  ConstIterator element( A.begin(i) );
761 
762  const size_t nonzeros( A.nonZeros(i) );
763  const size_t kpos( nonzeros & size_t(-4) );
764  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
765 
766  for( size_t k=0UL; k<kpos; k+=4UL )
767  {
768  const size_t i1( element->index() );
769  const ET1 v1( element->value() );
770  ++element;
771  const size_t i2( element->index() );
772  const ET1 v2( element->value() );
773  ++element;
774  const size_t i3( element->index() );
775  const ET1 v3( element->value() );
776  ++element;
777  const size_t i4( element->index() );
778  const ET1 v4( element->value() );
779  ++element;
780 
781  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
782 
783  const SIMDType xmm1( set( v1 ) );
784  const SIMDType xmm2( set( v2 ) );
785  const SIMDType xmm3( set( v3 ) );
786  const SIMDType xmm4( set( v4 ) );
787 
788  const size_t jbegin( ( IsUpper<MT5>::value )
790  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
791  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
792  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
793  const size_t jend( ( IsLower<MT5>::value )
795  ?( SYM || HERM || LOW ? min(i+1UL,i4) : i4 )
796  :( SYM || HERM || LOW ? min(i,i4)+1UL : i4+1UL ) )
797  :( SYM || HERM || LOW ? i+1UL : B.columns() ) );
798  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
799 
800  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
801  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
802 
803  size_t j( jbegin );
804 
805  for( ; j<jpos; j+=SIMDSIZE ) {
806  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) + xmm2 * B.load(i2,j) + xmm3 * B.load(i3,j) + xmm4 * B.load(i4,j) );
807  }
808  for( ; remainder && j<jend; ++j ) {
809  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
810  }
811  }
812 
813  for( ; element!=end; ++element )
814  {
815  const size_t i1( element->index() );
816  const ET1 v1( element->value() );
817 
818  const SIMDType xmm1( set( v1 ) );
819 
820  const size_t jbegin( ( IsUpper<MT5>::value )
822  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
823  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
824  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
825  const size_t jend( ( IsLower<MT5>::value )
827  ?( SYM || HERM || LOW ? min(i+1UL,i1) : i1 )
828  :( SYM || HERM || LOW ? min(i,i1)+1UL : i1+1UL ) )
829  :( SYM || HERM || LOW ? i+1UL : B.columns() ) );
830  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
831 
832  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
833  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
834 
835  size_t j( jbegin );
836 
837  for( ; j<jpos; j+=SIMDSIZE ) {
838  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) );
839  }
840  for( ; remainder && j<jend; ++j ) {
841  C(i,j) += v1 * B(i1,j);
842  }
843  }
844  }
845 
846  if( SYM || HERM ) {
847  for( size_t i=0UL; i<A.rows(); ++i ) {
848  for( size_t j=i+1UL; j<B.columns(); ++j ) {
849  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
850  }
851  }
852  }
853  }
855  //**********************************************************************************************
856 
857  //**Assignment to sparse matrices***************************************************************
870  template< typename MT // Type of the target sparse matrix
871  , bool SO > // Storage order of the target sparse matrix
872  friend inline void assign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
873  {
875 
877 
884 
885  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
886  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
887 
888  const ForwardFunctor fwd;
889 
890  const TmpType tmp( serial( rhs ) );
891  assign( ~lhs, fwd( tmp ) );
892  }
894  //**********************************************************************************************
895 
896  //**Addition assignment to dense matrices*******************************************************
909  template< typename MT // Type of the target dense matrix
910  , bool SO > // Storage order of the target dense matrix
911  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
912  {
914 
915  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
916  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
917 
918  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
919  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
920 
921  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
922  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
923  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
924  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
925  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
926  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
927 
928  SMatDMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
929  }
931  //**********************************************************************************************
932 
933  //**Default addition assignment to dense matrices***********************************************
947  template< typename MT3 // Type of the left-hand side target matrix
948  , typename MT4 // Type of the left-hand side matrix operand
949  , typename MT5 > // Type of the right-hand side matrix operand
951  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
952  {
954 
955  const size_t block( Or< IsRowMajorMatrix<MT3>, IsDiagonal<MT5> >::value ? B.columns() : 64UL );
956 
957  for( size_t jj=0UL; jj<B.columns(); jj+=block )
958  {
959  const size_t jtmp( min( jj+block, B.columns() ) );
960 
961  for( size_t i=0UL; i<A.rows(); ++i )
962  {
963  const ConstIterator end( A.end(i) );
964  ConstIterator element( A.begin(i) );
965 
966  for( ; element!=end; ++element )
967  {
968  const size_t i1( element->index() );
969 
971  {
972  C(i,i1) += element->value() * B(i1,i1);
973  }
974  else
975  {
976  const size_t jbegin( ( IsUpper<MT5>::value )
977  ?( ( UPP )
978  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
979  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
980  :( jj ) );
981  const size_t jend( ( IsLower<MT5>::value )
982  ?( ( LOW )
983  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
984  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
985  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
986 
987  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
988  continue;
989 
990  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
991 
992  const size_t jnum( jend - jbegin );
993  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
994  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
995 
996  for( size_t j=jbegin; j<jpos; j+=4UL ) {
997  C(i,j ) += element->value() * B(i1,j );
998  C(i,j+1UL) += element->value() * B(i1,j+1UL);
999  C(i,j+2UL) += element->value() * B(i1,j+2UL);
1000  C(i,j+3UL) += element->value() * B(i1,j+3UL);
1001  }
1002  for( size_t j=jpos; j<jend; ++j ) {
1003  C(i,j) += element->value() * B(i1,j);
1004  }
1005  }
1006  }
1007  }
1008  }
1009  }
1011  //**********************************************************************************************
1012 
1013  //**Optimized addition assignment to dense matrices*********************************************
1027  template< typename MT3 // Type of the left-hand side target matrix
1028  , typename MT4 // Type of the left-hand side matrix operand
1029  , typename MT5 > // Type of the right-hand side matrix operand
1031  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1032  {
1034 
1035  const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 64UL );
1036 
1037  for( size_t jj=0UL; jj<B.columns(); jj+=block )
1038  {
1039  const size_t jtmp( min( jj+block, B.columns() ) );
1040 
1041  for( size_t i=0UL; i<A.rows(); ++i )
1042  {
1043  const ConstIterator end( A.end(i) );
1044  ConstIterator element( A.begin(i) );
1045 
1046  const size_t nonzeros( A.nonZeros(i) );
1047  const size_t kpos( nonzeros & size_t(-4) );
1048  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1049 
1050  for( size_t k=0UL; k<kpos; k+=4UL )
1051  {
1052  const size_t i1( element->index() );
1053  const ET1 v1( element->value() );
1054  ++element;
1055  const size_t i2( element->index() );
1056  const ET1 v2( element->value() );
1057  ++element;
1058  const size_t i3( element->index() );
1059  const ET1 v3( element->value() );
1060  ++element;
1061  const size_t i4( element->index() );
1062  const ET1 v4( element->value() );
1063  ++element;
1064 
1065  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1066 
1067  const size_t jbegin( ( IsUpper<MT5>::value )
1068  ?( ( UPP )
1069  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1070  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1071  :( UPP ? max(i,jj) : jj ) );
1072  const size_t jend( ( IsLower<MT5>::value )
1073  ?( ( LOW )
1074  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) )
1075  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) ) )
1076  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1077 
1078  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1079  continue;
1080 
1081  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1082 
1083  const size_t jnum( jend - jbegin );
1084  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1085  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1086 
1087  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1088  C(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1089  C(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1090  C(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1091  C(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1092  }
1093  for( size_t j=jpos; j<jend; ++j ) {
1094  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1095  }
1096  }
1097 
1098  for( ; element!=end; ++element )
1099  {
1100  const size_t i1( element->index() );
1101  const ET1 v1( element->value() );
1102 
1103  const size_t jbegin( ( IsUpper<MT5>::value )
1104  ?( ( UPP )
1105  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1106  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1107  :( UPP ? max(i,jj) : jj ) );
1108  const size_t jend( ( IsLower<MT5>::value )
1109  ?( ( LOW )
1110  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
1111  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
1112  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1113 
1114  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1115  continue;
1116 
1117  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1118 
1119  const size_t jnum( jend - jbegin );
1120  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1121  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1122 
1123  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1124  C(i,j ) += v1 * B(i1,j );
1125  C(i,j+1UL) += v1 * B(i1,j+1UL);
1126  C(i,j+2UL) += v1 * B(i1,j+2UL);
1127  C(i,j+3UL) += v1 * B(i1,j+3UL);
1128  }
1129  for( size_t j=jpos; j<jend; ++j ) {
1130  C(i,j) += v1 * B(i1,j);
1131  }
1132  }
1133  }
1134  }
1135  }
1137  //**********************************************************************************************
1138 
1139  //**Vectorized addition assignment to dense matrices********************************************
1153  template< typename MT3 // Type of the left-hand side target matrix
1154  , typename MT4 // Type of the left-hand side matrix operand
1155  , typename MT5 > // Type of the right-hand side matrix operand
1157  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1158  {
1160 
1161  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT5>::value );
1162 
1163  for( size_t i=0UL; i<A.rows(); ++i )
1164  {
1165  const ConstIterator end( A.end(i) );
1166  ConstIterator element( A.begin(i) );
1167 
1168  const size_t nonzeros( A.nonZeros(i) );
1169  const size_t kpos( nonzeros & size_t(-4) );
1170  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1171 
1172  for( size_t k=0UL; k<kpos; k+=4UL )
1173  {
1174  const size_t i1( element->index() );
1175  const ET1 v1( element->value() );
1176  ++element;
1177  const size_t i2( element->index() );
1178  const ET1 v2( element->value() );
1179  ++element;
1180  const size_t i3( element->index() );
1181  const ET1 v3( element->value() );
1182  ++element;
1183  const size_t i4( element->index() );
1184  const ET1 v4( element->value() );
1185  ++element;
1186 
1187  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1188 
1189  const SIMDType xmm1( set( v1 ) );
1190  const SIMDType xmm2( set( v2 ) );
1191  const SIMDType xmm3( set( v3 ) );
1192  const SIMDType xmm4( set( v4 ) );
1193 
1194  const size_t jbegin( ( IsUpper<MT5>::value )
1196  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
1197  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
1198  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
1199  const size_t jend( ( IsLower<MT5>::value )
1201  ?( LOW ? min(i+1UL,i4) : i4 )
1202  :( LOW ? min(i,i4)+1UL : i4+1UL ) )
1203  :( LOW ? i+1UL : B.columns() ) );
1204  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1205 
1206  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1207  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
1208 
1209  size_t j( jbegin );
1210 
1211  for( ; j<jpos; j+=SIMDSIZE ) {
1212  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) + xmm2 * B.load(i2,j) + xmm3 * B.load(i3,j) + xmm4 * B.load(i4,j) );
1213  }
1214  for( ; remainder && j<jend; ++j ) {
1215  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1216  }
1217  }
1218 
1219  for( ; element!=end; ++element )
1220  {
1221  const size_t i1( element->index() );
1222  const ET1 v1( element->value() );
1223 
1224  const SIMDType xmm1( set( v1 ) );
1225 
1226  const size_t jbegin( ( IsUpper<MT5>::value )
1228  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
1229  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
1230  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
1231  const size_t jend( ( IsLower<MT5>::value )
1233  ?( LOW ? min(i+1UL,i1) : i1 )
1234  :( LOW ? min(i,i1)+1UL : i1+1UL ) )
1235  :( LOW ? i+1UL : B.columns() ) );
1236  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1237 
1238  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1239  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
1240 
1241  size_t j( jbegin );
1242 
1243  for( ; j<jpos; j+=SIMDSIZE ) {
1244  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) );
1245  }
1246  for( ; remainder && j<jend; ++j ) {
1247  C(i,j) += v1 * B(i1,j);
1248  }
1249  }
1250  }
1251  }
1253  //**********************************************************************************************
1254 
1255  //**Addition assignment to sparse matrices******************************************************
1256  // No special implementation for the addition assignment to sparse matrices.
1257  //**********************************************************************************************
1258 
1259  //**Subtraction assignment to dense matrices****************************************************
1272  template< typename MT // Type of the target dense matrix
1273  , bool SO > // Storage order of the target dense matrix
1274  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1275  {
1277 
1278  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1279  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1280 
1281  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
1282  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
1283 
1284  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1285  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1286  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1287  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1288  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1289  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1290 
1291  SMatDMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1292  }
1294  //**********************************************************************************************
1295 
1296  //**Default subtraction assignment to dense matrices********************************************
1310  template< typename MT3 // Type of the left-hand side target matrix
1311  , typename MT4 // Type of the left-hand side matrix operand
1312  , typename MT5 > // Type of the right-hand side matrix operand
1314  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1315  {
1317 
1318  const size_t block( Or< IsRowMajorMatrix<MT3>, IsDiagonal<MT5> >::value ? B.columns() : 64UL );
1319 
1320  for( size_t jj=0UL; jj<B.columns(); jj+=block )
1321  {
1322  const size_t jtmp( min( jj+block, B.columns() ) );
1323 
1324  for( size_t i=0UL; i<A.rows(); ++i )
1325  {
1326  const ConstIterator end( A.end(i) );
1327  ConstIterator element( A.begin(i) );
1328 
1329  for( ; element!=end; ++element )
1330  {
1331  const size_t i1( element->index() );
1332 
1334  {
1335  C(i,i1) -= element->value() * B(i1,i1);
1336  }
1337  else
1338  {
1339  const size_t jbegin( ( IsUpper<MT5>::value )
1340  ?( ( UPP )
1341  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1342  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1343  :( jj ) );
1344  const size_t jend( ( IsLower<MT5>::value )
1345  ?( ( LOW )
1346  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
1347  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
1348  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1349 
1350  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1351  continue;
1352 
1353  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1354 
1355  const size_t jnum( jend - jbegin );
1356  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1357  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1358 
1359  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1360  C(i,j ) -= element->value() * B(i1,j );
1361  C(i,j+1UL) -= element->value() * B(i1,j+1UL);
1362  C(i,j+2UL) -= element->value() * B(i1,j+2UL);
1363  C(i,j+3UL) -= element->value() * B(i1,j+3UL);
1364  }
1365  for( size_t j=jpos; j<jend; ++j ) {
1366  C(i,j) -= element->value() * B(i1,j);
1367  }
1368  }
1369  }
1370  }
1371  }
1372  }
1374  //**********************************************************************************************
1375 
1376  //**Optimized subtraction assignment to dense matrices******************************************
1390  template< typename MT3 // Type of the left-hand side target matrix
1391  , typename MT4 // Type of the left-hand side matrix operand
1392  , typename MT5 > // Type of the right-hand side matrix operand
1394  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1395  {
1397 
1398  const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 64UL );
1399 
1400  for( size_t jj=0UL; jj<B.columns(); jj+=block )
1401  {
1402  const size_t jtmp( min( jj+block, B.columns() ) );
1403 
1404  for( size_t i=0UL; i<A.rows(); ++i )
1405  {
1406  const ConstIterator end( A.end(i) );
1407  ConstIterator element( A.begin(i) );
1408 
1409  const size_t nonzeros( A.nonZeros(i) );
1410  const size_t kpos( nonzeros & size_t(-4) );
1411  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1412 
1413  for( size_t k=0UL; k<kpos; k+=4UL )
1414  {
1415  const size_t i1( element->index() );
1416  const ET1 v1( element->value() );
1417  ++element;
1418  const size_t i2( element->index() );
1419  const ET1 v2( element->value() );
1420  ++element;
1421  const size_t i3( element->index() );
1422  const ET1 v3( element->value() );
1423  ++element;
1424  const size_t i4( element->index() );
1425  const ET1 v4( element->value() );
1426  ++element;
1427 
1428  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1429 
1430  const size_t jbegin( ( IsUpper<MT5>::value )
1431  ?( ( UPP )
1432  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1433  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1434  :( UPP ? max(i,jj) : jj ) );
1435  const size_t jend( ( IsLower<MT5>::value )
1436  ?( ( LOW )
1437  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) )
1438  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) ) )
1439  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1440 
1441  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1442  continue;
1443 
1444  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1445 
1446  const size_t jnum( jend - jbegin );
1447  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1448  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1449 
1450  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1451  C(i,j ) -= v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1452  C(i,j+1UL) -= v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1453  C(i,j+2UL) -= v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1454  C(i,j+3UL) -= v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1455  }
1456  for( size_t j=jpos; j<jend; ++j ) {
1457  C(i,j) -= v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1458  }
1459  }
1460 
1461  for( ; element!=end; ++element )
1462  {
1463  const size_t i1( element->index() );
1464  const ET1 v1( element->value() );
1465 
1466  const size_t jbegin( ( IsUpper<MT5>::value )
1467  ?( ( UPP )
1468  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1469  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1470  :( UPP ? max(i,jj) : jj ) );
1471  const size_t jend( ( IsLower<MT5>::value )
1472  ?( ( LOW )
1473  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
1474  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
1475  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1476 
1477  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1478  continue;
1479 
1480  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1481 
1482  const size_t jnum( jend - jbegin );
1483  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1484  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1485 
1486  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1487  C(i,j ) -= v1 * B(i1,j );
1488  C(i,j+1UL) -= v1 * B(i1,j+1UL);
1489  C(i,j+2UL) -= v1 * B(i1,j+2UL);
1490  C(i,j+3UL) -= v1 * B(i1,j+3UL);
1491  }
1492  for( size_t j=jpos; j<jend; ++j ) {
1493  C(i,j) -= v1 * B(i1,j);
1494  }
1495  }
1496  }
1497  }
1498  }
1500  //**********************************************************************************************
1501 
1502  //**Vectorized subtraction assignment to dense matrices*****************************************
1516  template< typename MT3 // Type of the left-hand side target matrix
1517  , typename MT4 // Type of the left-hand side matrix operand
1518  , typename MT5 > // Type of the right-hand side matrix operand
1520  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1521  {
1523 
1524  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT5>::value );
1525 
1526  for( size_t i=0UL; i<A.rows(); ++i )
1527  {
1528  const ConstIterator end( A.end(i) );
1529  ConstIterator element( A.begin(i) );
1530 
1531  const size_t nonzeros( A.nonZeros(i) );
1532  const size_t kpos( nonzeros & size_t(-4) );
1533  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1534 
1535  for( size_t k=0UL; k<kpos; k+=4UL )
1536  {
1537  const size_t i1( element->index() );
1538  const ET1 v1( element->value() );
1539  ++element;
1540  const size_t i2( element->index() );
1541  const ET1 v2( element->value() );
1542  ++element;
1543  const size_t i3( element->index() );
1544  const ET1 v3( element->value() );
1545  ++element;
1546  const size_t i4( element->index() );
1547  const ET1 v4( element->value() );
1548  ++element;
1549 
1550  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1551 
1552  const SIMDType xmm1( set( v1 ) );
1553  const SIMDType xmm2( set( v2 ) );
1554  const SIMDType xmm3( set( v3 ) );
1555  const SIMDType xmm4( set( v4 ) );
1556 
1557  const size_t jbegin( ( IsUpper<MT5>::value )
1559  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
1560  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
1561  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
1562  const size_t jend( ( IsLower<MT5>::value )
1564  ?( LOW ? min(i+1UL,i4) : i4 )
1565  :( LOW ? min(i,i4)+1UL : i4+1UL ) )
1566  :( LOW ? i+1UL : B.columns() ) );
1567  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1568 
1569  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1570  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
1571 
1572  size_t j( jbegin );
1573 
1574  for( ; j<jpos; j+=SIMDSIZE ) {
1575  C.store( i, j, C.load(i,j) - xmm1 * B.load(i1,j) - xmm2 * B.load(i2,j) - xmm3 * B.load(i3,j) - xmm4 * B.load(i4,j) );
1576  }
1577  for( ; remainder && j<jend; ++j ) {
1578  C(i,j) -= v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1579  }
1580  }
1581 
1582  for( ; element!=end; ++element )
1583  {
1584  const size_t i1( element->index() );
1585  const ET1 v1( element->value() );
1586 
1587  const SIMDType xmm1( set( v1 ) );
1588 
1589  const size_t jbegin( ( IsUpper<MT5>::value )
1591  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
1592  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
1593  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
1594  const size_t jend( ( IsLower<MT5>::value )
1596  ?( LOW ? min(i+1UL,i1) : i1 )
1597  :( LOW ? min(i,i1)+1UL : i1+1UL ) )
1598  :( LOW ? i+1UL : B.columns() ) );
1599  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1600 
1601  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1602  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
1603 
1604  size_t j( jbegin );
1605 
1606  for( ; j<jpos; j+=SIMDSIZE ) {
1607  C.store( i, j, C.load(i,j) - xmm1 * B.load(i1,j) );
1608  }
1609  for( ; remainder && j<jend; ++j ) {
1610  C(i,j) -= v1 * B(i1,j);
1611  }
1612  }
1613  }
1614  }
1616  //**********************************************************************************************
1617 
1618  //**Subtraction assignment to sparse matrices***************************************************
1619  // No special implementation for the subtraction assignment to sparse matrices.
1620  //**********************************************************************************************
1621 
1622  //**Multiplication assignment to dense matrices*************************************************
1623  // No special implementation for the multiplication assignment to dense matrices.
1624  //**********************************************************************************************
1625 
1626  //**Multiplication assignment to sparse matrices************************************************
1627  // No special implementation for the multiplication assignment to sparse matrices.
1628  //**********************************************************************************************
1629 
1630  //**SMP assignment to dense matrices************************************************************
1645  template< typename MT // Type of the target dense matrix
1646  , bool SO > // Storage order of the target dense matrix
1648  smpAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1649  {
1651 
1652  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1653  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1654 
1655  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1656  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1657 
1658  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1659  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1660  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1661  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1662  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1663  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1664 
1665  smpAssign( ~lhs, A * B );
1666  }
1668  //**********************************************************************************************
1669 
1670  //**SMP assignment to sparse matrices***********************************************************
1685  template< typename MT // Type of the target sparse matrix
1686  , bool SO > // Storage order of the target sparse matrix
1688  smpAssign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1689  {
1691 
1693 
1700 
1701  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1702  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1703 
1704  const ForwardFunctor fwd;
1705 
1706  const TmpType tmp( rhs );
1707  smpAssign( ~lhs, fwd( tmp ) );
1708  }
1710  //**********************************************************************************************
1711 
1712  //**SMP addition assignment to dense matrices***************************************************
1728  template< typename MT // Type of the target dense matrix
1729  , bool SO > // Storage order of the target dense matrix
1732  {
1734 
1735  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1736  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1737 
1738  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1739  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1740 
1741  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1742  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1743  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1744  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1745  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1746  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1747 
1748  smpAddAssign( ~lhs, A * B );
1749  }
1751  //**********************************************************************************************
1752 
1753  //**SMP addition assignment to sparse matrices**************************************************
1754  // No special implementation for the SMP addition assignment to sparse matrices.
1755  //**********************************************************************************************
1756 
1757  //**SMP subtraction assignment to dense matrices************************************************
1773  template< typename MT // Type of the target dense matrix
1774  , bool SO > // Storage order of the target dense matrix
1777  {
1779 
1780  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1781  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1782 
1783  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1784  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1785 
1786  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1787  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1788  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1789  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1790  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1791  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1792 
1793  smpSubAssign( ~lhs, A * B );
1794  }
1796  //**********************************************************************************************
1797 
1798  //**SMP subtraction assignment to sparse matrices***********************************************
1799  // No special implementation for the SMP subtraction assignment to sparse matrices.
1800  //**********************************************************************************************
1801 
1802  //**SMP multiplication assignment to dense matrices*********************************************
1803  // No special implementation for the SMP multiplication assignment to dense matrices.
1804  //**********************************************************************************************
1805 
1806  //**SMP multiplication assignment to sparse matrices********************************************
1807  // No special implementation for the SMP multiplication assignment to sparse matrices.
1808  //**********************************************************************************************
1809 
1810  //**Compile time checks*************************************************************************
1818  //**********************************************************************************************
1819 };
1820 //*************************************************************************************************
1821 
1822 
1823 
1824 
1825 //=================================================================================================
1826 //
1827 // GLOBAL BINARY ARITHMETIC OPERATORS
1828 //
1829 //=================================================================================================
1830 
1831 //*************************************************************************************************
1860 template< typename T1 // Type of the left-hand side sparse matrix
1861  , typename T2 > // Type of the right-hand side dense matrix
1864 {
1866 
1867  if( (~lhs).columns() != (~rhs).rows() ) {
1868  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1869  }
1870 
1872 }
1873 //*************************************************************************************************
1874 
1875 
1876 
1877 
1878 //=================================================================================================
1879 //
1880 // GLOBAL FUNCTIONS
1881 //
1882 //=================================================================================================
1883 
1884 //*************************************************************************************************
1908 template< typename MT1 // Type of the left-hand side sparse matrix
1909  , typename MT2 // Type of the right-hand side dense matrix
1910  , bool SF // Symmetry flag
1911  , bool HF // Hermitian flag
1912  , bool LF // Lower flag
1913  , bool UF > // Upper flag
1916 {
1918 
1919  if( !isSquare( dm ) ) {
1920  BLAZE_THROW_INVALID_ARGUMENT( "Invalid symmetric matrix specification" );
1921  }
1922 
1923  return SMatDMatMultExpr<MT1,MT2,true,HF,LF,UF>( dm.leftOperand(), dm.rightOperand() );
1924 }
1926 //*************************************************************************************************
1927 
1928 
1929 //*************************************************************************************************
1953 template< typename MT1 // Type of the left-hand side sparse matrix
1954  , typename MT2 // Type of the right-hand side dense matrix
1955  , bool SF // Symmetry flag
1956  , bool HF // Hermitian flag
1957  , bool LF // Lower flag
1958  , bool UF > // Upper flag
1961 {
1963 
1964  if( !isSquare( dm ) ) {
1965  BLAZE_THROW_INVALID_ARGUMENT( "Invalid Hermitian matrix specification" );
1966  }
1967 
1968  return SMatDMatMultExpr<MT1,MT2,SF,true,LF,UF>( dm.leftOperand(), dm.rightOperand() );
1969 }
1971 //*************************************************************************************************
1972 
1973 
1974 //*************************************************************************************************
1998 template< typename MT1 // Type of the left-hand side dense matrix
1999  , typename MT2 // Type of the right-hand side dense matrix
2000  , bool SF // Symmetry flag
2001  , bool HF // Hermitian flag
2002  , bool LF // Lower flag
2003  , bool UF > // Upper flag
2006 {
2008 
2009  if( !isSquare( dm ) ) {
2010  BLAZE_THROW_INVALID_ARGUMENT( "Invalid lower matrix specification" );
2011  }
2012 
2013  return SMatDMatMultExpr<MT1,MT2,SF,HF,true,UF>( dm.leftOperand(), dm.rightOperand() );
2014 }
2016 //*************************************************************************************************
2017 
2018 
2019 //*************************************************************************************************
2043 template< typename MT1 // Type of the left-hand side dense matrix
2044  , typename MT2 // Type of the right-hand side dense matrix
2045  , bool SF // Symmetry flag
2046  , bool HF // Hermitian flag
2047  , bool LF // Lower flag
2048  , bool UF > // Upper flag
2051 {
2053 
2054  if( !isSquare( dm ) ) {
2055  BLAZE_THROW_INVALID_ARGUMENT( "Invalid upper matrix specification" );
2056  }
2057 
2058  return SMatDMatMultExpr<MT1,MT2,SF,HF,LF,true>( dm.leftOperand(), dm.rightOperand() );
2059 }
2061 //*************************************************************************************************
2062 
2063 
2064 //*************************************************************************************************
2088 template< typename MT1 // Type of the left-hand side dense matrix
2089  , typename MT2 // Type of the right-hand side dense matrix
2090  , bool SF // Symmetry flag
2091  , bool HF // Hermitian flag
2092  , bool LF // Lower flag
2093  , bool UF > // Upper flag
2096 {
2098 
2099  if( !isSquare( dm ) ) {
2100  BLAZE_THROW_INVALID_ARGUMENT( "Invalid diagonal matrix specification" );
2101  }
2102 
2103  return SMatDMatMultExpr<MT1,MT2,SF,HF,true,true>( dm.leftOperand(), dm.rightOperand() );
2104 }
2106 //*************************************************************************************************
2107 
2108 
2109 
2110 
2111 //=================================================================================================
2112 //
2113 // ROWS SPECIALIZATIONS
2114 //
2115 //=================================================================================================
2116 
2117 //*************************************************************************************************
2119 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2120 struct Rows< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> > : public Rows<MT1>
2121 {};
2123 //*************************************************************************************************
2124 
2125 
2126 
2127 
2128 //=================================================================================================
2129 //
2130 // COLUMNS SPECIALIZATIONS
2131 //
2132 //=================================================================================================
2133 
2134 //*************************************************************************************************
2136 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2137 struct Columns< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> > : public Columns<MT2>
2138 {};
2140 //*************************************************************************************************
2141 
2142 
2143 
2144 
2145 //=================================================================================================
2146 //
2147 // ISALIGNED SPECIALIZATIONS
2148 //
2149 //=================================================================================================
2150 
2151 //*************************************************************************************************
2153 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2154 struct IsAligned< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2155  : public BoolConstant< IsAligned<MT2>::value >
2156 {};
2158 //*************************************************************************************************
2159 
2160 
2161 
2162 
2163 //=================================================================================================
2164 //
2165 // ISSYMMETRIC SPECIALIZATIONS
2166 //
2167 //=================================================================================================
2168 
2169 //*************************************************************************************************
2171 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2172 struct IsSymmetric< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2173  : public BoolConstant< Or< Bool<SF>
2174  , And< Bool<HF>
2175  , IsBuiltin< ElementType_< SMatDMatMultExpr<MT1,MT2,false,true,false,false> > > >
2176  , And< Bool<LF>, Bool<UF> > >::value >
2177 {};
2179 //*************************************************************************************************
2180 
2181 
2182 
2183 
2184 //=================================================================================================
2185 //
2186 // ISHERMITIAN SPECIALIZATIONS
2187 //
2188 //=================================================================================================
2189 
2190 //*************************************************************************************************
2192 template< typename MT1, typename MT2, bool SF, bool LF, bool UF >
2193 struct IsHermitian< SMatDMatMultExpr<MT1,MT2,SF,true,LF,UF> >
2194  : public TrueType
2195 {};
2197 //*************************************************************************************************
2198 
2199 
2200 
2201 
2202 //=================================================================================================
2203 //
2204 // ISLOWER SPECIALIZATIONS
2205 //
2206 //=================================================================================================
2207 
2208 //*************************************************************************************************
2210 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2211 struct IsLower< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2212  : public BoolConstant< Or< Bool<LF>
2213  , And< IsLower<MT1>, IsLower<MT2> >
2214  , And< Or< Bool<SF>, Bool<HF> >
2215  , IsUpper<MT1>, IsUpper<MT2> > >::value >
2216 {};
2218 //*************************************************************************************************
2219 
2220 
2221 
2222 
2223 //=================================================================================================
2224 //
2225 // ISUNILOWER SPECIALIZATIONS
2226 //
2227 //=================================================================================================
2228 
2229 //*************************************************************************************************
2231 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2232 struct IsUniLower< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2233  : public BoolConstant< Or< And< IsUniLower<MT1>, IsUniLower<MT2> >
2234  , And< Or< Bool<SF>, Bool<HF> >
2235  , IsUniUpper<MT1>, IsUniUpper<MT2> > >::value >
2236 {};
2238 //*************************************************************************************************
2239 
2240 
2241 
2242 
2243 //=================================================================================================
2244 //
2245 // ISSTRICTLYLOWER SPECIALIZATIONS
2246 //
2247 //=================================================================================================
2248 
2249 //*************************************************************************************************
2251 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2252 struct IsStrictlyLower< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2253  : public BoolConstant< Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2254  , And< IsStrictlyLower<MT2>, IsLower<MT1> >
2255  , And< Or< Bool<SF>, Bool<HF> >
2256  , Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2257  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > > > >::value >
2258 {};
2260 //*************************************************************************************************
2261 
2262 
2263 
2264 
2265 //=================================================================================================
2266 //
2267 // ISUPPER SPECIALIZATIONS
2268 //
2269 //=================================================================================================
2270 
2271 //*************************************************************************************************
2273 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2274 struct IsUpper< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2275  : public BoolConstant< Or< Bool<UF>
2276  , And< IsUpper<MT1>, IsUpper<MT2> >
2277  , And< Or< Bool<SF>, Bool<HF> >
2278  , IsLower<MT1>, IsLower<MT2> > >::value >
2279 {};
2281 //*************************************************************************************************
2282 
2283 
2284 
2285 
2286 //=================================================================================================
2287 //
2288 // ISUNIUPPER SPECIALIZATIONS
2289 //
2290 //=================================================================================================
2291 
2292 //*************************************************************************************************
2294 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2295 struct IsUniUpper< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2296  : public BoolConstant< Or< And< IsUniUpper<MT1>, IsUniUpper<MT2> >
2297  , And< Or< Bool<SF>, Bool<HF> >
2298  , IsUniLower<MT1>, IsUniLower<MT2> > >::value >
2299 {};
2301 //*************************************************************************************************
2302 
2303 
2304 
2305 
2306 //=================================================================================================
2307 //
2308 // ISSTRICTLYUPPER SPECIALIZATIONS
2309 //
2310 //=================================================================================================
2311 
2312 //*************************************************************************************************
2314 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2315 struct IsStrictlyUpper< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2316  : public BoolConstant< Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2317  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> >
2318  , And< Or< Bool<SF>, Bool<HF> >
2319  , Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2320  , And< IsStrictlyLower<MT2>, IsLower<MT1> > > > >::value >
2321 {};
2323 //*************************************************************************************************
2324 
2325 
2326 
2327 
2328 //=================================================================================================
2329 //
2330 // EXPRESSION TRAIT SPECIALIZATIONS
2331 //
2332 //=================================================================================================
2333 
2334 //*************************************************************************************************
2336 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF, typename VT >
2337 struct DMatDVecMultExprTrait< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, VT >
2338 {
2339  public:
2340  //**********************************************************************************************
2345  , INVALID_TYPE >;
2346  //**********************************************************************************************
2347 };
2349 //*************************************************************************************************
2350 
2351 
2352 //*************************************************************************************************
2354 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF, typename VT >
2355 struct DMatSVecMultExprTrait< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, VT >
2356 {
2357  public:
2358  //**********************************************************************************************
2363  , INVALID_TYPE >;
2364  //**********************************************************************************************
2365 };
2367 //*************************************************************************************************
2368 
2369 
2370 //*************************************************************************************************
2372 template< typename VT, typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2373 struct TDVecDMatMultExprTrait< VT, SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2374 {
2375  public:
2376  //**********************************************************************************************
2381  , INVALID_TYPE >;
2382  //**********************************************************************************************
2383 };
2385 //*************************************************************************************************
2386 
2387 
2388 //*************************************************************************************************
2390 template< typename VT, typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2391 struct TSVecDMatMultExprTrait< VT, SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2392 {
2393  public:
2394  //**********************************************************************************************
2399  , INVALID_TYPE >;
2400  //**********************************************************************************************
2401 };
2403 //*************************************************************************************************
2404 
2405 
2406 //*************************************************************************************************
2408 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2409 struct DMatDeclSymExprTrait< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2410 {
2411  public:
2412  //**********************************************************************************************
2416  , INVALID_TYPE >;
2417  //**********************************************************************************************
2418 };
2420 //*************************************************************************************************
2421 
2422 
2423 //*************************************************************************************************
2425 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2426 struct DMatDeclHermExprTrait< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2427 {
2428  public:
2429  //**********************************************************************************************
2433  , INVALID_TYPE >;
2434  //**********************************************************************************************
2435 };
2437 //*************************************************************************************************
2438 
2439 
2440 //*************************************************************************************************
2442 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2443 struct DMatDeclLowExprTrait< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2444 {
2445  public:
2446  //**********************************************************************************************
2450  , INVALID_TYPE >;
2451  //**********************************************************************************************
2452 };
2454 //*************************************************************************************************
2455 
2456 
2457 //*************************************************************************************************
2459 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2460 struct DMatDeclUppExprTrait< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2461 {
2462  public:
2463  //**********************************************************************************************
2467  , INVALID_TYPE >;
2468  //**********************************************************************************************
2469 };
2471 //*************************************************************************************************
2472 
2473 
2474 //*************************************************************************************************
2476 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2477 struct DMatDeclDiagExprTrait< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2478 {
2479  public:
2480  //**********************************************************************************************
2484  , INVALID_TYPE >;
2485  //**********************************************************************************************
2486 };
2488 //*************************************************************************************************
2489 
2490 
2491 //*************************************************************************************************
2493 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF, bool AF >
2494 struct SubmatrixExprTrait< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, AF >
2495 {
2496  public:
2497  //**********************************************************************************************
2500  //**********************************************************************************************
2501 };
2503 //*************************************************************************************************
2504 
2505 
2506 //*************************************************************************************************
2508 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2509 struct RowExprTrait< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2510 {
2511  public:
2512  //**********************************************************************************************
2513  using Type = MultExprTrait_< RowExprTrait_<const MT1>, MT2 >;
2514  //**********************************************************************************************
2515 };
2517 //*************************************************************************************************
2518 
2519 
2520 //*************************************************************************************************
2522 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2523 struct ColumnExprTrait< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2524 {
2525  public:
2526  //**********************************************************************************************
2528  //**********************************************************************************************
2529 };
2531 //*************************************************************************************************
2532 
2533 } // namespace blaze
2534 
2535 #endif
typename SubmatrixExprTrait< MT, AF >::Type SubmatrixExprTrait_
Auxiliary alias declaration for the SubmatrixExprTrait type trait.The SubmatrixExprTrait_ alias decla...
Definition: SubmatrixExprTrait.h:134
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Evaluation of the expression type of a dense matrix declupp operation.Via this type trait it is possi...
Definition: DMatDeclUppExprTrait.h:75
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: SMatDMatMultExpr.h:464
Compile time check for row vector types.This type trait tests whether or not the given template argum...
Definition: IsRowVector.h:80
const DMatForEachExpr< MT, Conj, SO > conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition: DMatForEachExpr.h:1214
Header file for auxiliary alias declarations.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:72
Header file for mathematical functions.
constexpr bool useOptimizedKernels
Configuration switch for optimized kernels.This configuration switch enables/disables all optimized c...
Definition: Optimizations.h:84
Header file for the SMatDVecMultExprTrait class template.
Header file for the Rows type trait.
Header file for the IsUniUpper type trait.
ResultType_< MT2 > RT2
Result type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:153
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:87
Header file for basic type definitions.
Header file for the DMatDeclDiagExprTrait class template.
typename TSVecDMatMultExprTrait< VT, MT >::Type TSVecDMatMultExprTrait_
Auxiliary alias declaration for the TSVecDMatMultExprTrait class template.The TSVecDMatMultExprTrait_...
Definition: TSVecDMatMultExprTrait.h:119
MultTrait_< RT1, RT2 > ResultType
Result type for expression template evaluations.
Definition: SMatDMatMultExpr.h:267
EnableIf_< IsDenseMatrix< MT1 > > smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
Header file for the IsSparseMatrix type trait.
Header file for the serial shim.
Header file for the IsDiagonal type trait.
Generic wrapper for a compile time constant integral value.The IntegralConstant class template repres...
Definition: IntegralConstant.h:71
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the ColumnExprTrait class template.
Header file for the DeclUpp functor.
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:194
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:162
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:315
SMatDMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the SMatDMatMultExpr class.
Definition: SMatDMatMultExpr.h:311
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:533
Flag for symmetric matrices.
Definition: SMatDMatMultExpr.h:173
Header file for the IsRowVector type trait.
Header file for the And class template.
const ElementType_< MT > min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1755
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:88
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:163
RightOperand rightOperand() const noexcept
Returns the right-hand side dense matrix operand.
Definition: SMatDMatMultExpr.h:420
Header file for the TDVecSMatMultExprTrait class template.
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:721
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:245
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Expression object for sparse matrix-dense matrix multiplications.The SMatDMatMultExpr class represent...
Definition: Forward.h:100
DisableIf_< IsSymmetric< MT >, const DMatDeclSymExpr< MT, SO > > declsym(const DenseMatrix< MT, SO > &dm)
Declares the given non-symmetric dense matrix expression dm as symmetric.
Definition: DMatDeclSymExpr.h:841
Evaluation of the expression type of a dense matrix/dense vector multiplication.Via this type trait i...
Definition: DMatDVecMultExprTrait.h:78
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:88
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Header file for the TSVecSMatMultExprTrait class template.
Compile time check for data types.This type trait tests whether or not the given types can be combine...
Definition: IsSIMDCombinable.h:120
Header file for the IsUniLower type trait.
If_< IsExpression< MT2 >, const MT2, const MT2 &> RightOperand
Composite type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:279
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:323
const ElementType_< MT > max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1802
EnableIf_< IsDenseMatrix< MT1 > > smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
CompositeType_< MT1 > CT1
Composite type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:156
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:71
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:119
const ElementType ReturnType
Return type for expression template evaluations.
Definition: SMatDMatMultExpr.h:272
Constraint on the data type.
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
Compile time check for the alignment of data types.This type trait tests whether the given data type ...
Definition: IsAligned.h:87
Evaluation of the expression type of a dense matrix declsym operation.Via this type trait it is possi...
Definition: DMatDeclSymExprTrait.h:75
Constraint on the data type.
Constraint on the data type.
typename MultExprTrait< T1, T2 >::Type MultExprTrait_
Auxiliary alias declaration for the MultExprTrait class template.The MultExprTrait_ alias declaration...
Definition: MultExprTrait.h:344
Header file for the MultExprTrait class template.
DisableIf_< IsHermitian< MT >, const DMatDeclHermExpr< MT, SO > > declherm(const DenseMatrix< MT, SO > &dm)
Declares the given non-Hermitian dense matrix expression dm as Hermitian.
Definition: DMatDeclHermExpr.h:841
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:72
SubvectorExprTrait_< VT, unaligned > subvector(Vector< VT, TF > &vector, size_t index, size_t size)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:152
LeftOperand leftOperand() const noexcept
Returns the left-hand side sparse matrix operand.
Definition: SMatDMatMultExpr.h:410
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: SMatDMatMultExpr.h:432
Compile time check for upper unitriangular matrices.This type trait tests whether or not the given te...
Definition: IsUniUpper.h:86
Compile time check for dense vector types.This type trait tests whether or not the given template par...
Definition: IsDenseVector.h:78
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
Header file for the DeclLow functor.
Header file for the If class template.
Compile time check for row-major matrix types.This type trait tests whether or not the given template...
Definition: IsRowMajorMatrix.h:83
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2939
Generic wrapper for the decllow() function.
Definition: DeclLow.h:58
Compile time check for data types with padding.This type trait tests whether the given data type empl...
Definition: IsPadded.h:76
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Header file for the Or class template.
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: SMatDMatMultExpr.h:374
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
Header file for the DenseMatrix base class.
Header file for the DMatDeclLowExprTrait class template.
Header file for the Columns type trait.
Flag for lower matrices.
Definition: SMatDMatMultExpr.h:175
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
Evaluation of the expression type of a dense matrix/sparse vector multiplication.Via this type trait ...
Definition: DMatSVecMultExprTrait.h:80
ElementType_< RT1 > ET1
Element type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:154
Compile time check for sparse vector types.This type trait tests whether or not the given template pa...
Definition: IsSparseVector.h:78
Evaluation of the expression type type of a submatrix operation.Via this type trait it is possible to...
Definition: SubmatrixExprTrait.h:80
IfTrue_< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side sparse matrix operand.
Definition: SMatDMatMultExpr.h:282
Header file for the DMatDVecMultExprTrait class template.
Header file for the IsLower type trait.
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:90
DisableIf_< Or< IsComputation< MT >, IsTransExpr< MT >, IsDeclExpr< MT > >, RowExprTrait_< MT > > row(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific row of the given matrix.
Definition: Row.h:128
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:60
Generic wrapper for the null function.
Definition: Noop.h:58
Header file for the IsTriangular type trait.
Header file for the DMatDeclUppExprTrait class template.
Header file for the DMatDeclSymExprTrait class template.
Compile time check for column vector types.This type trait tests whether or not the given template ar...
Definition: IsColumnVector.h:80
Constraints on the storage order of matrix types.
typename TDVecDMatMultExprTrait< VT, MT >::Type TDVecDMatMultExprTrait_
Auxiliary alias declaration for the TDVecDMatMultExprTrait class template.The TDVecDMatMultExprTrait_...
Definition: TDVecDMatMultExprTrait.h:119
Compile time check for symmetric matrices.This type trait tests whether or not the given template par...
Definition: IsSymmetric.h:85
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: SMatDMatMultExpr.h:271
Header file for the exception macros of the math module.
DisableIf_< IsLower< MT >, const DMatDeclLowExpr< MT, SO > > decllow(const DenseMatrix< MT, SO > &dm)
Declares the given non-lower dense matrix expression dm as lower.
Definition: DMatDeclLowExpr.h:842
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
const ResultType CompositeType
Data type for composite expression templates.
Definition: SMatDMatMultExpr.h:273
Evaluation of the expression type type of a row operation.Via this type trait it is possible to evalu...
Definition: RowExprTrait.h:79
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:260
Header file for the DeclDiag functor.
Evaluation of the expression type of a dense matrix declherm operation.Via this type trait it is poss...
Definition: DMatDeclHermExprTrait.h:75
Compile time check for dense matrix types.This type trait tests whether or not the given template par...
Definition: IsDenseMatrix.h:78
Header file for the RowExprTrait class template.
Header file for all forward declarations for expression class templates.
Header file for the IsDenseMatrix type trait.
ResultType_< MT1 > RT1
Result type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:152
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: SMatDMatMultExpr.h:326
DisableIf_< Or< IsComputation< MT >, IsTransExpr< MT >, IsDeclExpr< MT > >, ColumnExprTrait_< MT > > column(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific column of the given matrix.
Definition: Column.h:128
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:109
Compile time check for lower unitriangular matrices.This type trait tests whether or not the given te...
Definition: IsUniLower.h:86
CompositeType_< MT2 > CT2
Composite type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:157
If_< IsExpression< MT1 >, const MT1, const MT1 &> LeftOperand
Composite type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:276
Header file for the conjugate shim.
Compile time check for resizable data types.This type trait tests whether the given data type is a re...
Definition: IsResizable.h:75
Header file for the IsSIMDCombinable type trait.
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: SMatDMatMultExpr.h:400
Header file for the IsSparseVector type trait.
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
Header file for run time assertion macros.
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: SMatDMatMultExpr.h:390
Utility type for generic codes.
LeftOperand lhs_
Left-hand side sparse matrix of the multiplication expression.
Definition: SMatDMatMultExpr.h:471
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:160
Header file for the reset shim.
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:296
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:93
Header file for the DMatDeclHermExprTrait class template.
Compile time check for Hermitian matrices.This type trait tests whether or not the given template par...
Definition: IsHermitian.h:85
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: SMatDMatMultExpr.h:444
Constraints on the storage order of matrix types.
Generic wrapper for the declherm() function.
Definition: DeclHerm.h:58
typename SMatDVecMultExprTrait< MT, VT >::Type SMatDVecMultExprTrait_
Auxiliary alias declaration for the SMatDVecMultExprTrait class template.The SMatDVecMultExprTrait_ a...
Definition: SMatDVecMultExprTrait.h:119
Header file for the Noop functor.
Header file for the RemoveReference type trait.
OppositeType_< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: SMatDMatMultExpr.h:268
Flag for upper matrices.
Definition: SMatDMatMultExpr.h:176
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:223
typename T::OppositeType OppositeType_
Alias declaration for nested OppositeType type definitions.The OppositeType_ alias declaration provid...
Definition: Aliases.h:243
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:84
Header file for the IsDenseVector type trait.
Generic wrapper for the declupp() function.
Definition: DeclUpp.h:58
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
SMatDMatMultExpr< MT1, MT2, SF, HF, LF, UF > This
Type of this SMatDMatMultExpr instance.
Definition: SMatDMatMultExpr.h:265
Evaluation of the expression type of a dense vector/dense matrix multiplication.Via this type trait i...
Definition: TDVecDMatMultExprTrait.h:78
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: SMatDMatMultExpr.h:454
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: SMatDMatMultExpr.h:269
Evaluation of the expression type of a sparse vector/dense matrix multiplication.Via this type trait ...
Definition: TSVecDMatMultExprTrait.h:78
Header file for the IsRowMajorMatrix type trait.
Header file for the IsComputation type trait class.
Header file for the IsBuiltin type trait.
Header file for the TDVecDMatMultExprTrait class template.
Compile time logical or evaluation.The Or class template performs at compile time a logical or (&#39;&&&#39;)...
Definition: Or.h:101
Header file for the IntegralConstant class template.
Compile time evaluation of the number of columns of a matrix.The Columns type trait evaluates the num...
Definition: Columns.h:76
Evaluation of the expression type of a dense matrix decllow operation.Via this type trait it is possi...
Definition: DMatDeclLowExprTrait.h:75
Generic wrapper for the decldiag() function.
Definition: DeclDiag.h:58
Evaluation of the expression type of a dense matrix decldiag operation.Via this type trait it is poss...
Definition: DMatDeclDiagExprTrait.h:75
Compile time evaluation of the number of rows of a matrix.The Rows type trait evaluates the number of...
Definition: Rows.h:76
Header file for the TSVecDMatMultExprTrait class template.
Compile time check for sparse matrix types.This type trait tests whether or not the given template pa...
Definition: IsSparseMatrix.h:78
Header file for the DeclHerm functor.
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:573
ElementType_< RT2 > ET2
Element type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:155
IfTrue_< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: SMatDMatMultExpr.h:285
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:403
Header file for the IsUpper type trait.
Header file for the DMatSVecMultExprTrait class template.
Header file for the IsColumnVector type trait.
Constraint on the data type.
Generic wrapper for the declsym() function.
Definition: DeclSym.h:58
BLAZE_ALWAYS_INLINE bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition: Matrix.h:677
Header file for the IsResizable type trait.
const DMatDMatMultExpr< T1, T2, false, false, false, false > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:7505
Flag for Hermitian matrices.
Definition: SMatDMatMultExpr.h:174
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
DisableIf_< IsDiagonal< MT >, const DMatDeclDiagExpr< MT, SO > > decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given non-diagonal dense matrix expression dm as diagonal.
Definition: DMatDeclDiagExpr.h:841
DisableIf_< IsUpper< MT >, const DMatDeclUppExpr< MT, SO > > declupp(const DenseMatrix< MT, SO > &dm)
Declares the given non-upper dense matrix expression dm as upper.
Definition: DMatDeclUppExpr.h:842
Evaluation of the expression type type of a column operation.Via this type trait it is possible to ev...
Definition: ColumnExprTrait.h:78
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the Bool class template.
Header file for the DeclSym functor.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:61
Header file for the IsExpression type trait class.
Header file for the function trace functionality.
RightOperand rhs_
Right-hand side dense matrix of the multiplication expression.
Definition: SMatDMatMultExpr.h:472
ElementType_< ResultType > ElementType
Resulting element type.
Definition: SMatDMatMultExpr.h:270