SMatDMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
51 #include <blaze/math/Exception.h>
63 #include <blaze/math/shims/Reset.h>
65 #include <blaze/math/SIMD.h>
90 #include <blaze/util/Assert.h>
91 #include <blaze/util/EnableIf.h>
94 #include <blaze/util/InvalidType.h>
95 #include <blaze/util/mpl/And.h>
96 #include <blaze/util/mpl/Bool.h>
97 #include <blaze/util/mpl/If.h>
98 #include <blaze/util/mpl/Or.h>
99 #include <blaze/util/Types.h>
102 
103 
104 namespace blaze {
105 
106 //=================================================================================================
107 //
108 // CLASS SMATDMATMULTEXPR
109 //
110 //=================================================================================================
111 
112 //*************************************************************************************************
119 template< typename MT1 // Type of the left-hand side sparse matrix
120  , typename MT2 // Type of the right-hand side dense matrix
121  , bool SF // Symmetry flag
122  , bool HF // Hermitian flag
123  , bool LF // Lower flag
124  , bool UF > // Upper flag
125 class SMatDMatMultExpr
126  : public MatMatMultExpr< DenseMatrix< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, false > >
127  , private Computation
128 {
129  private:
130  //**Type definitions****************************************************************************
137  //**********************************************************************************************
138 
139  //**********************************************************************************************
141  enum : bool { evaluateLeft = IsComputation<MT1>::value || RequiresEvaluation<MT1>::value };
142  //**********************************************************************************************
143 
144  //**********************************************************************************************
146  enum : bool { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
147  //**********************************************************************************************
148 
149  //**********************************************************************************************
151  enum : bool {
152  SYM = ( SF && !( HF || LF || UF ) ),
153  HERM = ( HF && !( LF || UF ) ),
154  LOW = ( LF || ( ( SF || HF ) && UF ) ),
155  UPP = ( UF || ( ( SF || HF ) && LF ) )
156  };
157  //**********************************************************************************************
158 
159  //**********************************************************************************************
161 
165  template< typename T1, typename T2, typename T3 >
166  struct IsEvaluationRequired {
167  enum : bool { value = ( evaluateLeft || evaluateRight ) };
168  };
170  //**********************************************************************************************
171 
172  //**********************************************************************************************
174 
177  template< typename T1, typename T2, typename T3 >
178  struct UseVectorizedKernel {
179  enum : bool { value = useOptimizedKernels &&
181  T1::simdEnabled && T3::simdEnabled &&
185  , ElementType_<T3> >::value &&
188  };
190  //**********************************************************************************************
191 
192  //**********************************************************************************************
194 
198  template< typename T1, typename T2, typename T3 >
199  struct UseOptimizedKernel {
200  enum : bool { value = useOptimizedKernels &&
201  !UseVectorizedKernel<T1,T2,T3>::value &&
203  !IsResizable< ElementType_<T1> >::value &&
205  };
207  //**********************************************************************************************
208 
209  //**********************************************************************************************
211 
214  template< typename T1, typename T2, typename T3 >
215  struct UseDefaultKernel {
216  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
217  !UseOptimizedKernel<T1,T2,T3>::value };
218  };
220  //**********************************************************************************************
221 
222  //**********************************************************************************************
224 
227  using ForwardFunctor = IfTrue_< HERM
228  , DeclHerm
229  , IfTrue_< SYM
230  , DeclSym
231  , IfTrue_< LOW
232  , IfTrue_< UPP
233  , DeclDiag
234  , DeclLow >
235  , IfTrue_< UPP
236  , DeclUpp
237  , Noop > > > >;
239  //**********************************************************************************************
240 
241  public:
242  //**Type definitions****************************************************************************
245 
251  using ReturnType = const ElementType;
252  using CompositeType = const ResultType;
253 
255  using LeftOperand = If_< IsExpression<MT1>, const MT1, const MT1& >;
256 
258  using RightOperand = If_< IsExpression<MT2>, const MT2, const MT2& >;
259 
262 
265  //**********************************************************************************************
266 
267  //**Compilation flags***************************************************************************
269  enum : bool { simdEnabled = !IsDiagonal<MT2>::value &&
270  MT2::simdEnabled &&
273 
275  enum : bool { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
276  !evaluateRight && MT2::smpAssignable };
277  //**********************************************************************************************
278 
279  //**SIMD properties*****************************************************************************
281  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
282  //**********************************************************************************************
283 
284  //**Constructor*********************************************************************************
290  explicit inline SMatDMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
291  : lhs_( lhs ) // Left-hand side sparse matrix of the multiplication expression
292  , rhs_( rhs ) // Right-hand side dense matrix of the multiplication expression
293  {
294  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
295  }
296  //**********************************************************************************************
297 
298  //**Access operator*****************************************************************************
305  inline ReturnType operator()( size_t i, size_t j ) const {
306  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
307  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
308 
309  if( IsDiagonal<MT1>::value ) {
310  return lhs_(i,i) * rhs_(i,j);
311  }
312  else if( IsDiagonal<MT2>::value ) {
313  return lhs_(i,j) * rhs_(j,j);
314  }
316  const size_t begin( ( IsUpper<MT1>::value )
317  ?( ( IsLower<MT2>::value )
318  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
319  , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
320  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
321  :( ( IsLower<MT2>::value )
322  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
323  :( 0UL ) ) );
324  const size_t end( ( IsLower<MT1>::value )
325  ?( ( IsUpper<MT2>::value )
326  ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
327  , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
328  :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
329  :( ( IsUpper<MT2>::value )
330  ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
331  :( lhs_.columns() ) ) );
332 
333  if( begin >= end ) return ElementType();
334 
335  const size_t n( end - begin );
336 
337  return subvector( row( lhs_, i ), begin, n ) * subvector( column( rhs_, j ), begin, n );
338  }
339  else {
340  return row( lhs_, i ) * column( rhs_, j );
341  }
342  }
343  //**********************************************************************************************
344 
345  //**At function*********************************************************************************
353  inline ReturnType at( size_t i, size_t j ) const {
354  if( i >= lhs_.rows() ) {
355  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
356  }
357  if( j >= rhs_.columns() ) {
358  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
359  }
360  return (*this)(i,j);
361  }
362  //**********************************************************************************************
363 
364  //**Rows function*******************************************************************************
369  inline size_t rows() const noexcept {
370  return lhs_.rows();
371  }
372  //**********************************************************************************************
373 
374  //**Columns function****************************************************************************
379  inline size_t columns() const noexcept {
380  return rhs_.columns();
381  }
382  //**********************************************************************************************
383 
384  //**Left operand access*************************************************************************
389  inline LeftOperand leftOperand() const noexcept {
390  return lhs_;
391  }
392  //**********************************************************************************************
393 
394  //**Right operand access************************************************************************
399  inline RightOperand rightOperand() const noexcept {
400  return rhs_;
401  }
402  //**********************************************************************************************
403 
404  //**********************************************************************************************
410  template< typename T >
411  inline bool canAlias( const T* alias ) const noexcept {
412  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
413  }
414  //**********************************************************************************************
415 
416  //**********************************************************************************************
422  template< typename T >
423  inline bool isAliased( const T* alias ) const noexcept {
424  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
425  }
426  //**********************************************************************************************
427 
428  //**********************************************************************************************
433  inline bool isAligned() const noexcept {
434  return rhs_.isAligned();
435  }
436  //**********************************************************************************************
437 
438  //**********************************************************************************************
443  inline bool canSMPAssign() const noexcept {
444  return ( rows() * columns() >= SMP_SMATDMATMULT_THRESHOLD ) && !IsDiagonal<MT2>::value;
445  }
446  //**********************************************************************************************
447 
448  private:
449  //**Member variables****************************************************************************
452  //**********************************************************************************************
453 
454  //**Assignment to dense matrices****************************************************************
467  template< typename MT // Type of the target dense matrix
468  , bool SO > // Storage order of the target dense matrix
469  friend inline void assign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
470  {
472 
473  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
474  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
475 
476  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
477  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
478 
479  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
480  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
481  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
482  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
483  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
484  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
485 
486  SMatDMatMultExpr::selectAssignKernel( ~lhs, A, B );
487  }
489  //**********************************************************************************************
490 
491  //**Default assignment to dense matrices********************************************************
505  template< typename MT3 // Type of the left-hand side target matrix
506  , typename MT4 // Type of the left-hand side matrix operand
507  , typename MT5 > // Type of the right-hand side matrix operand
509  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
510  {
512 
513  const size_t block( Or< IsRowMajorMatrix<MT3>, IsDiagonal<MT5> >::value ? B.columns() : 64UL );
514 
515  reset( C );
516 
517  for( size_t jj=0UL; jj<B.columns(); jj+=block )
518  {
519  const size_t jtmp( min( jj+block, B.columns() ) );
520 
521  for( size_t i=0UL; i<A.rows(); ++i )
522  {
523  ConstIterator element( A.begin(i) );
524  const ConstIterator end( A.end(i) );
525 
526  for( ; element!=end; ++element )
527  {
528  const size_t i1( element->index() );
529 
531  {
532  C(i,i1) = element->value() * B(i1,i1);
533  }
534  else
535  {
536  const size_t jbegin( ( IsUpper<MT5>::value )
537  ?( ( UPP )
538  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
539  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
540  :( jj ) );
541  const size_t jend( ( IsLower<MT5>::value )
542  ?( ( SYM || HERM || LOW )
543  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
544  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
545  :( SYM || HERM || LOW ? min(i+1UL,jtmp) : jtmp ) );
546 
547  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
548  continue;
549 
550  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
551 
552  for( size_t j=jbegin; j<jend; ++j ) {
553  if( isDefault( C(i,j) ) )
554  C(i,j) = element->value() * B(i1,j);
555  else
556  C(i,j) += element->value() * B(i1,j);
557  }
558  }
559  }
560  }
561  }
562 
563  if( SYM || HERM ) {
564  for( size_t i=0UL; i<A.rows(); ++i ) {
565  for( size_t j=i+1UL; j<B.columns(); ++j ) {
566  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
567  }
568  }
569  }
570  }
572  //**********************************************************************************************
573 
574  //**Optimized assignment to dense matrices******************************************************
588  template< typename MT3 // Type of the left-hand side target matrix
589  , typename MT4 // Type of the left-hand side matrix operand
590  , typename MT5 > // Type of the right-hand side matrix operand
592  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
593  {
595 
596  const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 64UL );
597 
598  reset( C );
599 
600  for( size_t jj=0UL; jj<B.columns(); jj+=block )
601  {
602  const size_t jtmp( min( jj+block, B.columns() ) );
603 
604  for( size_t i=0UL; i<A.rows(); ++i )
605  {
606  const ConstIterator end( A.end(i) );
607  ConstIterator element( A.begin(i) );
608 
609  const size_t nonzeros( A.nonZeros(i) );
610  const size_t kpos( nonzeros & size_t(-4) );
611  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
612 
613  for( size_t k=0UL; k<kpos; k+=4UL )
614  {
615  const size_t i1( element->index() );
616  const ET1 v1( element->value() );
617  ++element;
618  const size_t i2( element->index() );
619  const ET1 v2( element->value() );
620  ++element;
621  const size_t i3( element->index() );
622  const ET1 v3( element->value() );
623  ++element;
624  const size_t i4( element->index() );
625  const ET1 v4( element->value() );
626  ++element;
627 
628  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
629 
630  const size_t jbegin( ( IsUpper<MT5>::value )
631  ?( ( UPP )
632  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
633  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
634  :( UPP ? max(i,jj) : jj ) );
635  const size_t jend( ( IsLower<MT5>::value )
636  ?( ( SYM || HERM || LOW )
637  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) )
638  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) ) )
639  :( SYM || HERM || LOW ? min(i+1UL,jtmp) : jtmp ) );
640 
641  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
642  continue;
643 
644  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
645 
646  const size_t jnum( jend - jbegin );
647  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
648  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
649 
650  for( size_t j=jbegin; j<jpos; j+=4UL ) {
651  C(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
652  C(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
653  C(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
654  C(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
655  }
656  for( size_t j=jpos; j<jend; ++j ) {
657  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
658  }
659  }
660 
661  for( ; element!=end; ++element )
662  {
663  const size_t i1( element->index() );
664  const ET1 v1( element->value() );
665 
666  const size_t jbegin( ( IsUpper<MT5>::value )
667  ?( ( UPP )
668  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
669  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
670  :( UPP ? max(i,jj) : jj ) );
671  const size_t jend( ( IsLower<MT5>::value )
672  ?( ( SYM || HERM || LOW )
673  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
674  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
675  :( SYM || HERM || LOW ? min(i+1UL,jtmp) : jtmp ) );
676 
677  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
678  continue;
679 
680  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
681 
682  const size_t jnum( jend - jbegin );
683  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
684  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
685 
686  for( size_t j=jbegin; j<jpos; j+=4UL ) {
687  C(i,j ) += v1 * B(i1,j );
688  C(i,j+1UL) += v1 * B(i1,j+1UL);
689  C(i,j+2UL) += v1 * B(i1,j+2UL);
690  C(i,j+3UL) += v1 * B(i1,j+3UL);
691  }
692  for( size_t j=jpos; j<jend; ++j ) {
693  C(i,j) += v1 * B(i1,j);
694  }
695  }
696  }
697  }
698 
699  if( SYM || HERM ) {
700  for( size_t i=0UL; i<A.rows(); ++i ) {
701  for( size_t j=i+1UL; j<B.columns(); ++j ) {
702  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
703  }
704  }
705  }
706  }
708  //**********************************************************************************************
709 
710  //**Vectorized assignment to dense matrices*****************************************************
724  template< typename MT3 // Type of the left-hand side target matrix
725  , typename MT4 // Type of the left-hand side matrix operand
726  , typename MT5 > // Type of the right-hand side matrix operand
728  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
729  {
731 
732  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT5>::value );
733 
734  reset( C );
735 
736  for( size_t i=0UL; i<A.rows(); ++i )
737  {
738  const ConstIterator end( A.end(i) );
739  ConstIterator element( A.begin(i) );
740 
741  const size_t nonzeros( A.nonZeros(i) );
742  const size_t kpos( nonzeros & size_t(-4) );
743  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
744 
745  for( size_t k=0UL; k<kpos; k+=4UL )
746  {
747  const size_t i1( element->index() );
748  const ET1 v1( element->value() );
749  ++element;
750  const size_t i2( element->index() );
751  const ET1 v2( element->value() );
752  ++element;
753  const size_t i3( element->index() );
754  const ET1 v3( element->value() );
755  ++element;
756  const size_t i4( element->index() );
757  const ET1 v4( element->value() );
758  ++element;
759 
760  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
761 
762  const SIMDType xmm1( set( v1 ) );
763  const SIMDType xmm2( set( v2 ) );
764  const SIMDType xmm3( set( v3 ) );
765  const SIMDType xmm4( set( v4 ) );
766 
767  const size_t jbegin( ( IsUpper<MT5>::value )
769  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
770  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
771  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
772  const size_t jend( ( IsLower<MT5>::value )
774  ?( SYM || HERM || LOW ? min(i+1UL,i4) : i4 )
775  :( SYM || HERM || LOW ? min(i,i4)+1UL : i4+1UL ) )
776  :( SYM || HERM || LOW ? i+1UL : B.columns() ) );
777  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
778 
779  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
780  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
781 
782  size_t j( jbegin );
783 
784  for( ; j<jpos; j+=SIMDSIZE ) {
785  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) + xmm2 * B.load(i2,j) + xmm3 * B.load(i3,j) + xmm4 * B.load(i4,j) );
786  }
787  for( ; remainder && j<jend; ++j ) {
788  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
789  }
790  }
791 
792  for( ; element!=end; ++element )
793  {
794  const size_t i1( element->index() );
795  const ET1 v1( element->value() );
796 
797  const SIMDType xmm1( set( v1 ) );
798 
799  const size_t jbegin( ( IsUpper<MT5>::value )
801  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
802  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
803  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
804  const size_t jend( ( IsLower<MT5>::value )
806  ?( SYM || HERM || LOW ? min(i+1UL,i1) : i1 )
807  :( SYM || HERM || LOW ? min(i,i1)+1UL : i1+1UL ) )
808  :( SYM || HERM || LOW ? i+1UL : B.columns() ) );
809  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
810 
811  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
812  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
813 
814  size_t j( jbegin );
815 
816  for( ; j<jpos; j+=SIMDSIZE ) {
817  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) );
818  }
819  for( ; remainder && j<jend; ++j ) {
820  C(i,j) += v1 * B(i1,j);
821  }
822  }
823  }
824 
825  if( SYM || HERM ) {
826  for( size_t i=0UL; i<A.rows(); ++i ) {
827  for( size_t j=i+1UL; j<B.columns(); ++j ) {
828  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
829  }
830  }
831  }
832  }
834  //**********************************************************************************************
835 
836  //**Assignment to sparse matrices***************************************************************
849  template< typename MT // Type of the target sparse matrix
850  , bool SO > // Storage order of the target sparse matrix
851  friend inline void assign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
852  {
854 
856 
863 
864  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
865  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
866 
867  const ForwardFunctor fwd;
868 
869  const TmpType tmp( serial( rhs ) );
870  assign( ~lhs, fwd( tmp ) );
871  }
873  //**********************************************************************************************
874 
875  //**Addition assignment to dense matrices*******************************************************
888  template< typename MT // Type of the target dense matrix
889  , bool SO > // Storage order of the target dense matrix
890  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
891  {
893 
894  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
895  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
896 
897  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
898  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
899 
900  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
901  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
902  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
903  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
904  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
905  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
906 
907  SMatDMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
908  }
910  //**********************************************************************************************
911 
912  //**Default addition assignment to dense matrices***********************************************
926  template< typename MT3 // Type of the left-hand side target matrix
927  , typename MT4 // Type of the left-hand side matrix operand
928  , typename MT5 > // Type of the right-hand side matrix operand
930  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
931  {
933 
934  const size_t block( Or< IsRowMajorMatrix<MT3>, IsDiagonal<MT5> >::value ? B.columns() : 64UL );
935 
936  for( size_t jj=0UL; jj<B.columns(); jj+=block )
937  {
938  const size_t jtmp( min( jj+block, B.columns() ) );
939 
940  for( size_t i=0UL; i<A.rows(); ++i )
941  {
942  const ConstIterator end( A.end(i) );
943  ConstIterator element( A.begin(i) );
944 
945  for( ; element!=end; ++element )
946  {
947  const size_t i1( element->index() );
948 
950  {
951  C(i,i1) += element->value() * B(i1,i1);
952  }
953  else
954  {
955  const size_t jbegin( ( IsUpper<MT5>::value )
956  ?( ( UPP )
957  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
958  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
959  :( jj ) );
960  const size_t jend( ( IsLower<MT5>::value )
961  ?( ( LOW )
962  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
963  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
964  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
965 
966  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
967  continue;
968 
969  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
970 
971  const size_t jnum( jend - jbegin );
972  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
973  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
974 
975  for( size_t j=jbegin; j<jpos; j+=4UL ) {
976  C(i,j ) += element->value() * B(i1,j );
977  C(i,j+1UL) += element->value() * B(i1,j+1UL);
978  C(i,j+2UL) += element->value() * B(i1,j+2UL);
979  C(i,j+3UL) += element->value() * B(i1,j+3UL);
980  }
981  for( size_t j=jpos; j<jend; ++j ) {
982  C(i,j) += element->value() * B(i1,j);
983  }
984  }
985  }
986  }
987  }
988  }
990  //**********************************************************************************************
991 
992  //**Optimized addition assignment to dense matrices*********************************************
1006  template< typename MT3 // Type of the left-hand side target matrix
1007  , typename MT4 // Type of the left-hand side matrix operand
1008  , typename MT5 > // Type of the right-hand side matrix operand
1010  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1011  {
1013 
1014  const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 64UL );
1015 
1016  for( size_t jj=0UL; jj<B.columns(); jj+=block )
1017  {
1018  const size_t jtmp( min( jj+block, B.columns() ) );
1019 
1020  for( size_t i=0UL; i<A.rows(); ++i )
1021  {
1022  const ConstIterator end( A.end(i) );
1023  ConstIterator element( A.begin(i) );
1024 
1025  const size_t nonzeros( A.nonZeros(i) );
1026  const size_t kpos( nonzeros & size_t(-4) );
1027  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1028 
1029  for( size_t k=0UL; k<kpos; k+=4UL )
1030  {
1031  const size_t i1( element->index() );
1032  const ET1 v1( element->value() );
1033  ++element;
1034  const size_t i2( element->index() );
1035  const ET1 v2( element->value() );
1036  ++element;
1037  const size_t i3( element->index() );
1038  const ET1 v3( element->value() );
1039  ++element;
1040  const size_t i4( element->index() );
1041  const ET1 v4( element->value() );
1042  ++element;
1043 
1044  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1045 
1046  const size_t jbegin( ( IsUpper<MT5>::value )
1047  ?( ( UPP )
1048  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1049  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1050  :( UPP ? max(i,jj) : jj ) );
1051  const size_t jend( ( IsLower<MT5>::value )
1052  ?( ( LOW )
1053  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) )
1054  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) ) )
1055  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1056 
1057  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1058  continue;
1059 
1060  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1061 
1062  const size_t jnum( jend - jbegin );
1063  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1064  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1065 
1066  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1067  C(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1068  C(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1069  C(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1070  C(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1071  }
1072  for( size_t j=jpos; j<jend; ++j ) {
1073  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1074  }
1075  }
1076 
1077  for( ; element!=end; ++element )
1078  {
1079  const size_t i1( element->index() );
1080  const ET1 v1( element->value() );
1081 
1082  const size_t jbegin( ( IsUpper<MT5>::value )
1083  ?( ( UPP )
1084  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1085  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1086  :( UPP ? max(i,jj) : jj ) );
1087  const size_t jend( ( IsLower<MT5>::value )
1088  ?( ( LOW )
1089  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
1090  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
1091  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1092 
1093  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1094  continue;
1095 
1096  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1097 
1098  const size_t jnum( jend - jbegin );
1099  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1100  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1101 
1102  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1103  C(i,j ) += v1 * B(i1,j );
1104  C(i,j+1UL) += v1 * B(i1,j+1UL);
1105  C(i,j+2UL) += v1 * B(i1,j+2UL);
1106  C(i,j+3UL) += v1 * B(i1,j+3UL);
1107  }
1108  for( size_t j=jpos; j<jend; ++j ) {
1109  C(i,j) += v1 * B(i1,j);
1110  }
1111  }
1112  }
1113  }
1114  }
1116  //**********************************************************************************************
1117 
1118  //**Vectorized addition assignment to dense matrices********************************************
1132  template< typename MT3 // Type of the left-hand side target matrix
1133  , typename MT4 // Type of the left-hand side matrix operand
1134  , typename MT5 > // Type of the right-hand side matrix operand
1136  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1137  {
1139 
1140  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT5>::value );
1141 
1142  for( size_t i=0UL; i<A.rows(); ++i )
1143  {
1144  const ConstIterator end( A.end(i) );
1145  ConstIterator element( A.begin(i) );
1146 
1147  const size_t nonzeros( A.nonZeros(i) );
1148  const size_t kpos( nonzeros & size_t(-4) );
1149  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1150 
1151  for( size_t k=0UL; k<kpos; k+=4UL )
1152  {
1153  const size_t i1( element->index() );
1154  const ET1 v1( element->value() );
1155  ++element;
1156  const size_t i2( element->index() );
1157  const ET1 v2( element->value() );
1158  ++element;
1159  const size_t i3( element->index() );
1160  const ET1 v3( element->value() );
1161  ++element;
1162  const size_t i4( element->index() );
1163  const ET1 v4( element->value() );
1164  ++element;
1165 
1166  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1167 
1168  const SIMDType xmm1( set( v1 ) );
1169  const SIMDType xmm2( set( v2 ) );
1170  const SIMDType xmm3( set( v3 ) );
1171  const SIMDType xmm4( set( v4 ) );
1172 
1173  const size_t jbegin( ( IsUpper<MT5>::value )
1175  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
1176  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
1177  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
1178  const size_t jend( ( IsLower<MT5>::value )
1180  ?( LOW ? min(i+1UL,i4) : i4 )
1181  :( LOW ? min(i,i4)+1UL : i4+1UL ) )
1182  :( LOW ? i+1UL : B.columns() ) );
1183  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1184 
1185  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1186  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
1187 
1188  size_t j( jbegin );
1189 
1190  for( ; j<jpos; j+=SIMDSIZE ) {
1191  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) + xmm2 * B.load(i2,j) + xmm3 * B.load(i3,j) + xmm4 * B.load(i4,j) );
1192  }
1193  for( ; remainder && j<jend; ++j ) {
1194  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1195  }
1196  }
1197 
1198  for( ; element!=end; ++element )
1199  {
1200  const size_t i1( element->index() );
1201  const ET1 v1( element->value() );
1202 
1203  const SIMDType xmm1( set( v1 ) );
1204 
1205  const size_t jbegin( ( IsUpper<MT5>::value )
1207  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
1208  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
1209  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
1210  const size_t jend( ( IsLower<MT5>::value )
1212  ?( LOW ? min(i+1UL,i1) : i1 )
1213  :( LOW ? min(i,i1)+1UL : i1+1UL ) )
1214  :( LOW ? i+1UL : B.columns() ) );
1215  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1216 
1217  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1218  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
1219 
1220  size_t j( jbegin );
1221 
1222  for( ; j<jpos; j+=SIMDSIZE ) {
1223  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) );
1224  }
1225  for( ; remainder && j<jend; ++j ) {
1226  C(i,j) += v1 * B(i1,j);
1227  }
1228  }
1229  }
1230  }
1232  //**********************************************************************************************
1233 
1234  //**Addition assignment to sparse matrices******************************************************
1235  // No special implementation for the addition assignment to sparse matrices.
1236  //**********************************************************************************************
1237 
1238  //**Subtraction assignment to dense matrices****************************************************
1251  template< typename MT // Type of the target dense matrix
1252  , bool SO > // Storage order of the target dense matrix
1253  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1254  {
1256 
1257  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1258  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1259 
1260  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
1261  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
1262 
1263  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1264  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1265  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1266  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1267  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1268  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1269 
1270  SMatDMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1271  }
1273  //**********************************************************************************************
1274 
1275  //**Default subtraction assignment to dense matrices********************************************
1289  template< typename MT3 // Type of the left-hand side target matrix
1290  , typename MT4 // Type of the left-hand side matrix operand
1291  , typename MT5 > // Type of the right-hand side matrix operand
1293  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1294  {
1296 
1297  const size_t block( Or< IsRowMajorMatrix<MT3>, IsDiagonal<MT5> >::value ? B.columns() : 64UL );
1298 
1299  for( size_t jj=0UL; jj<B.columns(); jj+=block )
1300  {
1301  const size_t jtmp( min( jj+block, B.columns() ) );
1302 
1303  for( size_t i=0UL; i<A.rows(); ++i )
1304  {
1305  const ConstIterator end( A.end(i) );
1306  ConstIterator element( A.begin(i) );
1307 
1308  for( ; element!=end; ++element )
1309  {
1310  const size_t i1( element->index() );
1311 
1313  {
1314  C(i,i1) -= element->value() * B(i1,i1);
1315  }
1316  else
1317  {
1318  const size_t jbegin( ( IsUpper<MT5>::value )
1319  ?( ( UPP )
1320  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1321  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1322  :( jj ) );
1323  const size_t jend( ( IsLower<MT5>::value )
1324  ?( ( LOW )
1325  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
1326  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
1327  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1328 
1329  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1330  continue;
1331 
1332  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1333 
1334  const size_t jnum( jend - jbegin );
1335  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1336  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1337 
1338  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1339  C(i,j ) -= element->value() * B(i1,j );
1340  C(i,j+1UL) -= element->value() * B(i1,j+1UL);
1341  C(i,j+2UL) -= element->value() * B(i1,j+2UL);
1342  C(i,j+3UL) -= element->value() * B(i1,j+3UL);
1343  }
1344  for( size_t j=jpos; j<jend; ++j ) {
1345  C(i,j) -= element->value() * B(i1,j);
1346  }
1347  }
1348  }
1349  }
1350  }
1351  }
1353  //**********************************************************************************************
1354 
1355  //**Optimized subtraction assignment to dense matrices******************************************
1369  template< typename MT3 // Type of the left-hand side target matrix
1370  , typename MT4 // Type of the left-hand side matrix operand
1371  , typename MT5 > // Type of the right-hand side matrix operand
1373  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1374  {
1376 
1377  const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 64UL );
1378 
1379  for( size_t jj=0UL; jj<B.columns(); jj+=block )
1380  {
1381  const size_t jtmp( min( jj+block, B.columns() ) );
1382 
1383  for( size_t i=0UL; i<A.rows(); ++i )
1384  {
1385  const ConstIterator end( A.end(i) );
1386  ConstIterator element( A.begin(i) );
1387 
1388  const size_t nonzeros( A.nonZeros(i) );
1389  const size_t kpos( nonzeros & size_t(-4) );
1390  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1391 
1392  for( size_t k=0UL; k<kpos; k+=4UL )
1393  {
1394  const size_t i1( element->index() );
1395  const ET1 v1( element->value() );
1396  ++element;
1397  const size_t i2( element->index() );
1398  const ET1 v2( element->value() );
1399  ++element;
1400  const size_t i3( element->index() );
1401  const ET1 v3( element->value() );
1402  ++element;
1403  const size_t i4( element->index() );
1404  const ET1 v4( element->value() );
1405  ++element;
1406 
1407  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1408 
1409  const size_t jbegin( ( IsUpper<MT5>::value )
1410  ?( ( UPP )
1411  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1412  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1413  :( UPP ? max(i,jj) : jj ) );
1414  const size_t jend( ( IsLower<MT5>::value )
1415  ?( ( LOW )
1416  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) )
1417  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) ) )
1418  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1419 
1420  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1421  continue;
1422 
1423  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1424 
1425  const size_t jnum( jend - jbegin );
1426  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1427  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1428 
1429  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1430  C(i,j ) -= v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1431  C(i,j+1UL) -= v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1432  C(i,j+2UL) -= v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1433  C(i,j+3UL) -= v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1434  }
1435  for( size_t j=jpos; j<jend; ++j ) {
1436  C(i,j) -= v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1437  }
1438  }
1439 
1440  for( ; element!=end; ++element )
1441  {
1442  const size_t i1( element->index() );
1443  const ET1 v1( element->value() );
1444 
1445  const size_t jbegin( ( IsUpper<MT5>::value )
1446  ?( ( UPP )
1447  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1448  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1449  :( UPP ? max(i,jj) : jj ) );
1450  const size_t jend( ( IsLower<MT5>::value )
1451  ?( ( LOW )
1452  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
1453  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
1454  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1455 
1456  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1457  continue;
1458 
1459  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1460 
1461  const size_t jnum( jend - jbegin );
1462  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1463  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1464 
1465  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1466  C(i,j ) -= v1 * B(i1,j );
1467  C(i,j+1UL) -= v1 * B(i1,j+1UL);
1468  C(i,j+2UL) -= v1 * B(i1,j+2UL);
1469  C(i,j+3UL) -= v1 * B(i1,j+3UL);
1470  }
1471  for( size_t j=jpos; j<jend; ++j ) {
1472  C(i,j) -= v1 * B(i1,j);
1473  }
1474  }
1475  }
1476  }
1477  }
1479  //**********************************************************************************************
1480 
1481  //**Vectorized subtraction assignment to dense matrices*****************************************
1495  template< typename MT3 // Type of the left-hand side target matrix
1496  , typename MT4 // Type of the left-hand side matrix operand
1497  , typename MT5 > // Type of the right-hand side matrix operand
1499  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1500  {
1502 
1503  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT5>::value );
1504 
1505  for( size_t i=0UL; i<A.rows(); ++i )
1506  {
1507  const ConstIterator end( A.end(i) );
1508  ConstIterator element( A.begin(i) );
1509 
1510  const size_t nonzeros( A.nonZeros(i) );
1511  const size_t kpos( nonzeros & size_t(-4) );
1512  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1513 
1514  for( size_t k=0UL; k<kpos; k+=4UL )
1515  {
1516  const size_t i1( element->index() );
1517  const ET1 v1( element->value() );
1518  ++element;
1519  const size_t i2( element->index() );
1520  const ET1 v2( element->value() );
1521  ++element;
1522  const size_t i3( element->index() );
1523  const ET1 v3( element->value() );
1524  ++element;
1525  const size_t i4( element->index() );
1526  const ET1 v4( element->value() );
1527  ++element;
1528 
1529  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1530 
1531  const SIMDType xmm1( set( v1 ) );
1532  const SIMDType xmm2( set( v2 ) );
1533  const SIMDType xmm3( set( v3 ) );
1534  const SIMDType xmm4( set( v4 ) );
1535 
1536  const size_t jbegin( ( IsUpper<MT5>::value )
1538  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
1539  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
1540  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
1541  const size_t jend( ( IsLower<MT5>::value )
1543  ?( LOW ? min(i+1UL,i4) : i4 )
1544  :( LOW ? min(i,i4)+1UL : i4+1UL ) )
1545  :( LOW ? i+1UL : B.columns() ) );
1546  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1547 
1548  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1549  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
1550 
1551  size_t j( jbegin );
1552 
1553  for( ; j<jpos; j+=SIMDSIZE ) {
1554  C.store( i, j, C.load(i,j) - xmm1 * B.load(i1,j) - xmm2 * B.load(i2,j) - xmm3 * B.load(i3,j) - xmm4 * B.load(i4,j) );
1555  }
1556  for( ; remainder && j<jend; ++j ) {
1557  C(i,j) -= v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1558  }
1559  }
1560 
1561  for( ; element!=end; ++element )
1562  {
1563  const size_t i1( element->index() );
1564  const ET1 v1( element->value() );
1565 
1566  const SIMDType xmm1( set( v1 ) );
1567 
1568  const size_t jbegin( ( IsUpper<MT5>::value )
1570  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
1571  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
1572  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
1573  const size_t jend( ( IsLower<MT5>::value )
1575  ?( LOW ? min(i+1UL,i1) : i1 )
1576  :( LOW ? min(i,i1)+1UL : i1+1UL ) )
1577  :( LOW ? i+1UL : B.columns() ) );
1578  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1579 
1580  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1581  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
1582 
1583  size_t j( jbegin );
1584 
1585  for( ; j<jpos; j+=SIMDSIZE ) {
1586  C.store( i, j, C.load(i,j) - xmm1 * B.load(i1,j) );
1587  }
1588  for( ; remainder && j<jend; ++j ) {
1589  C(i,j) -= v1 * B(i1,j);
1590  }
1591  }
1592  }
1593  }
1595  //**********************************************************************************************
1596 
1597  //**Subtraction assignment to sparse matrices***************************************************
1598  // No special implementation for the subtraction assignment to sparse matrices.
1599  //**********************************************************************************************
1600 
1601  //**Schur product assignment to dense matrices**************************************************
1614  template< typename MT // Type of the target dense matrix
1615  , bool SO > // Storage order of the target dense matrix
1616  friend inline void schurAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1617  {
1619 
1623 
1624  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1625  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1626 
1627  const ResultType tmp( serial( rhs ) );
1628  schurAssign( ~lhs, tmp );
1629  }
1631  //**********************************************************************************************
1632 
1633  //**Schur product assignment to sparse matrices*************************************************
1634  // No special implementation for the Schur product assignment to sparse matrices.
1635  //**********************************************************************************************
1636 
1637  //**Multiplication assignment to dense matrices*************************************************
1638  // No special implementation for the multiplication assignment to dense matrices.
1639  //**********************************************************************************************
1640 
1641  //**Multiplication assignment to sparse matrices************************************************
1642  // No special implementation for the multiplication assignment to sparse matrices.
1643  //**********************************************************************************************
1644 
1645  //**SMP assignment to dense matrices************************************************************
1660  template< typename MT // Type of the target dense matrix
1661  , bool SO > // Storage order of the target dense matrix
1663  smpAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1664  {
1666 
1667  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1668  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1669 
1670  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1671  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1672 
1673  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1674  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1675  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1676  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1677  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1678  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1679 
1680  smpAssign( ~lhs, A * B );
1681  }
1683  //**********************************************************************************************
1684 
1685  //**SMP assignment to sparse matrices***********************************************************
1700  template< typename MT // Type of the target sparse matrix
1701  , bool SO > // Storage order of the target sparse matrix
1703  smpAssign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1704  {
1706 
1708 
1715 
1716  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1717  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1718 
1719  const ForwardFunctor fwd;
1720 
1721  const TmpType tmp( rhs );
1722  smpAssign( ~lhs, fwd( tmp ) );
1723  }
1725  //**********************************************************************************************
1726 
1727  //**SMP addition assignment to dense matrices***************************************************
1743  template< typename MT // Type of the target dense matrix
1744  , bool SO > // Storage order of the target dense matrix
1747  {
1749 
1750  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1751  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1752 
1753  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1754  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1755 
1756  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1757  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1758  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1759  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1760  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1761  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1762 
1763  smpAddAssign( ~lhs, A * B );
1764  }
1766  //**********************************************************************************************
1767 
1768  //**SMP addition assignment to sparse matrices**************************************************
1769  // No special implementation for the SMP addition assignment to sparse matrices.
1770  //**********************************************************************************************
1771 
1772  //**SMP subtraction assignment to dense matrices************************************************
1788  template< typename MT // Type of the target dense matrix
1789  , bool SO > // Storage order of the target dense matrix
1792  {
1794 
1795  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1796  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1797 
1798  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1799  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1800 
1801  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1802  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1803  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1804  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1805  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1806  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1807 
1808  smpSubAssign( ~lhs, A * B );
1809  }
1811  //**********************************************************************************************
1812 
1813  //**SMP subtraction assignment to sparse matrices***********************************************
1814  // No special implementation for the SMP subtraction assignment to sparse matrices.
1815  //**********************************************************************************************
1816 
1817  //**SMP Schur product assignment to dense matrices**********************************************
1830  template< typename MT // Type of the target dense matrix
1831  , bool SO > // Storage order of the target dense matrix
1832  friend inline void smpSchurAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1833  {
1835 
1839 
1840  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1841  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1842 
1843  const ResultType tmp( rhs );
1844  smpSchurAssign( ~lhs, tmp );
1845  }
1847  //**********************************************************************************************
1848 
1849  //**SMP Schur product assignment to sparse matrices*********************************************
1850  // No special implementation for the SMP Schur product assignment to sparse matrices.
1851  //**********************************************************************************************
1852 
1853  //**SMP multiplication assignment to dense matrices*********************************************
1854  // No special implementation for the SMP multiplication assignment to dense matrices.
1855  //**********************************************************************************************
1856 
1857  //**SMP multiplication assignment to sparse matrices********************************************
1858  // No special implementation for the SMP multiplication assignment to sparse matrices.
1859  //**********************************************************************************************
1860 
1861  //**Compile time checks*************************************************************************
1869  //**********************************************************************************************
1870 };
1871 //*************************************************************************************************
1872 
1873 
1874 
1875 
1876 //=================================================================================================
1877 //
1878 // GLOBAL BINARY ARITHMETIC OPERATORS
1879 //
1880 //=================================================================================================
1881 
1882 //*************************************************************************************************
1911 template< typename MT1 // Type of the left-hand side sparse matrix
1912  , typename MT2 > // Type of the right-hand side dense matrix
1913 inline decltype(auto)
1914  operator*( const SparseMatrix<MT1,false>& lhs, const DenseMatrix<MT2,false>& rhs )
1915 {
1917 
1918  if( (~lhs).columns() != (~rhs).rows() ) {
1919  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1920  }
1921 
1923  return ReturnType( ~lhs, ~rhs );
1924 }
1925 //*************************************************************************************************
1926 
1927 
1928 
1929 
1930 //=================================================================================================
1931 //
1932 // GLOBAL FUNCTIONS
1933 //
1934 //=================================================================================================
1935 
1936 //*************************************************************************************************
1960 template< typename MT1 // Type of the left-hand side sparse matrix
1961  , typename MT2 // Type of the right-hand side dense matrix
1962  , bool SF // Symmetry flag
1963  , bool HF // Hermitian flag
1964  , bool LF // Lower flag
1965  , bool UF > // Upper flag
1966 inline decltype(auto) declsym( const SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
1967 {
1969 
1970  if( !isSquare( dm ) ) {
1971  BLAZE_THROW_INVALID_ARGUMENT( "Invalid symmetric matrix specification" );
1972  }
1973 
1975  return ReturnType( dm.leftOperand(), dm.rightOperand() );
1976 }
1978 //*************************************************************************************************
1979 
1980 
1981 //*************************************************************************************************
2005 template< typename MT1 // Type of the left-hand side sparse matrix
2006  , typename MT2 // Type of the right-hand side dense matrix
2007  , bool SF // Symmetry flag
2008  , bool HF // Hermitian flag
2009  , bool LF // Lower flag
2010  , bool UF > // Upper flag
2011 inline decltype(auto) declherm( const SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2012 {
2014 
2015  if( !isSquare( dm ) ) {
2016  BLAZE_THROW_INVALID_ARGUMENT( "Invalid Hermitian matrix specification" );
2017  }
2018 
2020  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2021 }
2023 //*************************************************************************************************
2024 
2025 
2026 //*************************************************************************************************
2050 template< typename MT1 // Type of the left-hand side dense matrix
2051  , typename MT2 // Type of the right-hand side dense matrix
2052  , bool SF // Symmetry flag
2053  , bool HF // Hermitian flag
2054  , bool LF // Lower flag
2055  , bool UF > // Upper flag
2056 inline decltype(auto) decllow( const SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2057 {
2059 
2060  if( !isSquare( dm ) ) {
2061  BLAZE_THROW_INVALID_ARGUMENT( "Invalid lower matrix specification" );
2062  }
2063 
2065  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2066 }
2068 //*************************************************************************************************
2069 
2070 
2071 //*************************************************************************************************
2095 template< typename MT1 // Type of the left-hand side dense matrix
2096  , typename MT2 // Type of the right-hand side dense matrix
2097  , bool SF // Symmetry flag
2098  , bool HF // Hermitian flag
2099  , bool LF // Lower flag
2100  , bool UF > // Upper flag
2101 inline decltype(auto) declupp( const SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2102 {
2104 
2105  if( !isSquare( dm ) ) {
2106  BLAZE_THROW_INVALID_ARGUMENT( "Invalid upper matrix specification" );
2107  }
2108 
2110  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2111 }
2113 //*************************************************************************************************
2114 
2115 
2116 //*************************************************************************************************
2140 template< typename MT1 // Type of the left-hand side dense matrix
2141  , typename MT2 // Type of the right-hand side dense matrix
2142  , bool SF // Symmetry flag
2143  , bool HF // Hermitian flag
2144  , bool LF // Lower flag
2145  , bool UF > // Upper flag
2146 inline decltype(auto) decldiag( const SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2147 {
2149 
2150  if( !isSquare( dm ) ) {
2151  BLAZE_THROW_INVALID_ARGUMENT( "Invalid diagonal matrix specification" );
2152  }
2153 
2155  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2156 }
2158 //*************************************************************************************************
2159 
2160 
2161 
2162 
2163 //=================================================================================================
2164 //
2165 // ROWS SPECIALIZATIONS
2166 //
2167 //=================================================================================================
2168 
2169 //*************************************************************************************************
2171 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2172 struct Rows< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2173  : public Rows<MT1>
2174 {};
2176 //*************************************************************************************************
2177 
2178 
2179 
2180 
2181 //=================================================================================================
2182 //
2183 // COLUMNS SPECIALIZATIONS
2184 //
2185 //=================================================================================================
2186 
2187 //*************************************************************************************************
2189 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2190 struct Columns< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2191  : public Columns<MT2>
2192 {};
2194 //*************************************************************************************************
2195 
2196 
2197 
2198 
2199 //=================================================================================================
2200 //
2201 // ISALIGNED SPECIALIZATIONS
2202 //
2203 //=================================================================================================
2204 
2205 //*************************************************************************************************
2207 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2208 struct IsAligned< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2209  : public BoolConstant< IsAligned<MT2>::value >
2210 {};
2212 //*************************************************************************************************
2213 
2214 
2215 
2216 
2217 //=================================================================================================
2218 //
2219 // ISSYMMETRIC SPECIALIZATIONS
2220 //
2221 //=================================================================================================
2222 
2223 //*************************************************************************************************
2225 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2226 struct IsSymmetric< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2227  : public BoolConstant< Or< Bool<SF>
2228  , And< Bool<HF>
2229  , IsBuiltin< ElementType_< SMatDMatMultExpr<MT1,MT2,false,true,false,false> > > >
2230  , And< Bool<LF>, Bool<UF> > >::value >
2231 {};
2233 //*************************************************************************************************
2234 
2235 
2236 
2237 
2238 //=================================================================================================
2239 //
2240 // ISHERMITIAN SPECIALIZATIONS
2241 //
2242 //=================================================================================================
2243 
2244 //*************************************************************************************************
2246 template< typename MT1, typename MT2, bool SF, bool LF, bool UF >
2247 struct IsHermitian< SMatDMatMultExpr<MT1,MT2,SF,true,LF,UF> >
2248  : public TrueType
2249 {};
2251 //*************************************************************************************************
2252 
2253 
2254 
2255 
2256 //=================================================================================================
2257 //
2258 // ISLOWER SPECIALIZATIONS
2259 //
2260 //=================================================================================================
2261 
2262 //*************************************************************************************************
2264 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2265 struct IsLower< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2266  : public BoolConstant< Or< Bool<LF>
2267  , And< IsLower<MT1>, IsLower<MT2> >
2268  , And< Or< Bool<SF>, Bool<HF> >
2269  , IsUpper<MT1>, IsUpper<MT2> > >::value >
2270 {};
2272 //*************************************************************************************************
2273 
2274 
2275 
2276 
2277 //=================================================================================================
2278 //
2279 // ISUNILOWER SPECIALIZATIONS
2280 //
2281 //=================================================================================================
2282 
2283 //*************************************************************************************************
2285 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2286 struct IsUniLower< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2287  : public BoolConstant< Or< And< IsUniLower<MT1>, IsUniLower<MT2> >
2288  , And< Or< Bool<SF>, Bool<HF> >
2289  , IsUniUpper<MT1>, IsUniUpper<MT2> > >::value >
2290 {};
2292 //*************************************************************************************************
2293 
2294 
2295 
2296 
2297 //=================================================================================================
2298 //
2299 // ISSTRICTLYLOWER SPECIALIZATIONS
2300 //
2301 //=================================================================================================
2302 
2303 //*************************************************************************************************
2305 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2306 struct IsStrictlyLower< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2307  : public BoolConstant< Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2308  , And< IsStrictlyLower<MT2>, IsLower<MT1> >
2309  , And< Or< Bool<SF>, Bool<HF> >
2310  , Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2311  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > > > >::value >
2312 {};
2314 //*************************************************************************************************
2315 
2316 
2317 
2318 
2319 //=================================================================================================
2320 //
2321 // ISUPPER SPECIALIZATIONS
2322 //
2323 //=================================================================================================
2324 
2325 //*************************************************************************************************
2327 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2328 struct IsUpper< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2329  : public BoolConstant< Or< Bool<UF>
2330  , And< IsUpper<MT1>, IsUpper<MT2> >
2331  , And< Or< Bool<SF>, Bool<HF> >
2332  , IsLower<MT1>, IsLower<MT2> > >::value >
2333 {};
2335 //*************************************************************************************************
2336 
2337 
2338 
2339 
2340 //=================================================================================================
2341 //
2342 // ISUNIUPPER SPECIALIZATIONS
2343 //
2344 //=================================================================================================
2345 
2346 //*************************************************************************************************
2348 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2349 struct IsUniUpper< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2350  : public BoolConstant< Or< And< IsUniUpper<MT1>, IsUniUpper<MT2> >
2351  , And< Or< Bool<SF>, Bool<HF> >
2352  , IsUniLower<MT1>, IsUniLower<MT2> > >::value >
2353 {};
2355 //*************************************************************************************************
2356 
2357 
2358 
2359 
2360 //=================================================================================================
2361 //
2362 // ISSTRICTLYUPPER SPECIALIZATIONS
2363 //
2364 //=================================================================================================
2365 
2366 //*************************************************************************************************
2368 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2369 struct IsStrictlyUpper< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2370  : public BoolConstant< Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2371  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> >
2372  , And< Or< Bool<SF>, Bool<HF> >
2373  , Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2374  , And< IsStrictlyLower<MT2>, IsLower<MT1> > > > >::value >
2375 {};
2377 //*************************************************************************************************
2378 
2379 } // namespace blaze
2380 
2381 #endif
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: SMatDMatMultExpr.h:443
Header file for auxiliary alias declarations.
Headerfile for the generic min algorithm.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:72
decltype(auto) decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as diagonal.
Definition: DMatDeclDiagExpr.h:996
Header file for the Rows type trait.
Header file for the IsUniUpper type trait.
EnableIf_< IsDenseMatrix< MT1 > > smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:196
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:87
Header file for basic type definitions.
CompositeType_< MT2 > CT2
Composite type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:136
Flag for Hermitian matrices.
Definition: SMatDMatMultExpr.h:153
Subvector< VT, AF > subvector(Vector< VT, TF > &vector, size_t index, size_t size)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:322
ElementType_< RT2 > ET2
Element type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:134
EnableIf_< IsDenseMatrix< MT1 > > smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:164
Header file for the serial shim.
Header file for the IsDiagonal type trait.
Flag for upper matrices.
Definition: SMatDMatMultExpr.h:155
Generic wrapper for a compile time constant integral value.The IntegralConstant class template repres...
Definition: IntegralConstant.h:71
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the DeclUpp functor.
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:198
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:172
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:316
SMatDMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the SMatDMatMultExpr class.
Definition: SMatDMatMultExpr.h:290
ResultType_< MT1 > RT1
Result type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:131
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:560
Header file for the And class template.
const ElementType_< MT > min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1762
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:88
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:171
RightOperand rightOperand() const noexcept
Returns the right-hand side dense matrix operand.
Definition: SMatDMatMultExpr.h:399
decltype(auto) declupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as upper.
Definition: DMatDeclUppExpr.h:1027
Flag for symmetric matrices.
Definition: SMatDMatMultExpr.h:152
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:250
Column< MT > column(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific column of the given matrix.
Definition: Column.h:124
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Expression object for sparse matrix-dense matrix multiplications.The SMatDMatMultExpr class represent...
Definition: Forward.h:107
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:88
ElementType_< ResultType > ElementType
Resulting element type.
Definition: SMatDMatMultExpr.h:249
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
IfTrue_< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: SMatDMatMultExpr.h:264
Compile time check for data types.This type trait tests whether or not the given types can be combine...
Definition: IsSIMDCombinable.h:120
Header file for the IsUniLower type trait.
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:343
const ElementType_< MT > max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1809
EnableIf_< IsDenseMatrix< MT1 > > smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:133
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:78
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:129
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
Row< MT > row(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific row of the given matrix.
Definition: Row.h:124
Compile time check for the alignment of data types.This type trait tests whether the given data type ...
Definition: IsAligned.h:87
Constraint on the data type.
Constraint on the data type.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:72
LeftOperand leftOperand() const noexcept
Returns the left-hand side sparse matrix operand.
Definition: SMatDMatMultExpr.h:389
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: SMatDMatMultExpr.h:411
Compile time check for upper unitriangular matrices.This type trait tests whether or not the given te...
Definition: IsUniUpper.h:86
Headerfile for the generic max algorithm.
Flag for lower matrices.
Definition: SMatDMatMultExpr.h:154
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
Header file for the DeclLow functor.
ElementType_< RT1 > ET1
Element type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:133
OppositeType_< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: SMatDMatMultExpr.h:247
Header file for the If class template.
Compile time check for row-major matrix types.This type trait tests whether or not the given template...
Definition: IsRowMajorMatrix.h:110
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
ResultType_< MT2 > RT2
Result type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:132
Generic wrapper for the decllow() function.
Definition: DeclLow.h:58
Compile time check for data types with padding.This type trait tests whether the given data type empl...
Definition: IsPadded.h:76
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:102
Header file for the Or class template.
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: SMatDMatMultExpr.h:353
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
Header file for the DenseMatrix base class.
Header file for the Columns type trait.
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:3087
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
decltype(auto) decllow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as lower.
Definition: DMatDeclLowExpr.h:1027
Header file for the IsLower type trait.
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:90
Generic wrapper for the null function.
Definition: Noop.h:58
Header file for the IsTriangular type trait.
Constraints on the storage order of matrix types.
Compile time check for symmetric matrices.This type trait tests whether or not the given template par...
Definition: IsSymmetric.h:85
Header file for the exception macros of the math module.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:264
Header file for the DeclDiag functor.
Constraint on the data type.
Header file for all forward declarations for expression class templates.
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: SMatDMatMultExpr.h:305
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:108
Compile time check for lower unitriangular matrices.This type trait tests whether or not the given te...
Definition: IsUniLower.h:86
Header file for the conjugate shim.
MultTrait_< RT1, RT2 > ResultType
Result type for expression template evaluations.
Definition: SMatDMatMultExpr.h:246
Compile time check for resizable data types.This type trait tests whether the given data type is a re...
Definition: IsResizable.h:75
Header file for the IsSIMDCombinable type trait.
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: SMatDMatMultExpr.h:379
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
Header file for run time assertion macros.
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: SMatDMatMultExpr.h:369
Utility type for generic codes.
LeftOperand lhs_
Left-hand side sparse matrix of the multiplication expression.
Definition: SMatDMatMultExpr.h:450
const ResultType CompositeType
Data type for composite expression templates.
Definition: SMatDMatMultExpr.h:252
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:154
Header file for the reset shim.
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:296
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:94
decltype(auto) declsym(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as symmetric.
Definition: DMatDeclSymExpr.h:1029
Compile time check for Hermitian matrices.This type trait tests whether or not the given template par...
Definition: IsHermitian.h:85
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: SMatDMatMultExpr.h:250
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: SMatDMatMultExpr.h:423
Constraints on the storage order of matrix types.
Generic wrapper for the declherm() function.
Definition: DeclHerm.h:58
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:819
Header file for the Noop functor.
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
Header file for the RemoveReference type trait.
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:224
typename T::OppositeType OppositeType_
Alias declaration for nested OppositeType type definitions.The OppositeType_ alias declaration provid...
Definition: Aliases.h:263
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:84
Generic wrapper for the declupp() function.
Definition: DeclUpp.h:58
CompositeType_< MT1 > CT1
Composite type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:135
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
const Type & ReturnType
Return type for expression template evaluations.
Definition: CompressedMatrix.h:3082
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: SMatDMatMultExpr.h:433
decltype(auto) declherm(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as Hermitian.
Definition: DMatDeclHermExpr.h:1029
Header file for the IsRowMajorMatrix type trait.
Header file for the IsComputation type trait class.
Header file for the IsBuiltin type trait.
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: SMatDMatMultExpr.h:248
Compile time logical or evaluation.The Or alias declaration performs at compile time a logical or (&#39;&&&#3...
Definition: Or.h:76
IfTrue_< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side sparse matrix operand.
Definition: SMatDMatMultExpr.h:261
If_< IsExpression< MT2 >, const MT2, const MT2 &> RightOperand
Composite type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:258
Header file for the IntegralConstant class template.
Compile time evaluation of the number of columns of a matrix.The Columns type trait evaluates the num...
Definition: Columns.h:75
Generic wrapper for the decldiag() function.
Definition: DeclDiag.h:58
If_< IsExpression< MT1 >, const MT1, const MT1 &> LeftOperand
Composite type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:255
Compile time evaluation of the number of rows of a matrix.The Rows type trait evaluates the number of...
Definition: Rows.h:75
Header file for the DeclHerm functor.
const ElementType ReturnType
Return type for expression template evaluations.
Definition: SMatDMatMultExpr.h:251
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:600
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:423
Header file for the IsUpper type trait.
decltype(auto) conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition: DMatMapExpr.h:1321
Constraint on the data type.
Generic wrapper for the declsym() function.
Definition: DeclSym.h:58
BLAZE_ALWAYS_INLINE bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition: Matrix.h:742
Header file for the IsResizable type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the Bool class template.
Header file for the DeclSym functor.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:61
Header file for the IsExpression type trait class.
Header file for the function trace functionality.
RightOperand rhs_
Right-hand side dense matrix of the multiplication expression.
Definition: SMatDMatMultExpr.h:451