SMatDMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_SMATDMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
51 #include <blaze/math/Exception.h>
63 #include <blaze/math/shims/Reset.h>
65 #include <blaze/math/SIMD.h>
85 #include <blaze/math/views/Check.h>
90 #include <blaze/util/Assert.h>
91 #include <blaze/util/EnableIf.h>
93 #include <blaze/util/mpl/And.h>
94 #include <blaze/util/mpl/Bool.h>
95 #include <blaze/util/mpl/If.h>
96 #include <blaze/util/mpl/Or.h>
97 #include <blaze/util/Types.h>
100 
101 
102 namespace blaze {
103 
104 //=================================================================================================
105 //
106 // CLASS SMATDMATMULTEXPR
107 //
108 //=================================================================================================
109 
110 //*************************************************************************************************
117 template< typename MT1 // Type of the left-hand side sparse matrix
118  , typename MT2 // Type of the right-hand side dense matrix
119  , bool SF // Symmetry flag
120  , bool HF // Hermitian flag
121  , bool LF // Lower flag
122  , bool UF > // Upper flag
123 class SMatDMatMultExpr
124  : public MatMatMultExpr< DenseMatrix< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, false > >
125  , private Computation
126 {
127  private:
128  //**Type definitions****************************************************************************
135  //**********************************************************************************************
136 
137  //**********************************************************************************************
139  enum : bool { evaluateLeft = IsComputation<MT1>::value || RequiresEvaluation<MT1>::value };
140  //**********************************************************************************************
141 
142  //**********************************************************************************************
144  enum : bool { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
145  //**********************************************************************************************
146 
147  //**********************************************************************************************
149  enum : bool {
150  SYM = ( SF && !( HF || LF || UF ) ),
151  HERM = ( HF && !( LF || UF ) ),
152  LOW = ( LF || ( ( SF || HF ) && UF ) ),
153  UPP = ( UF || ( ( SF || HF ) && LF ) )
154  };
155  //**********************************************************************************************
156 
157  //**********************************************************************************************
159 
163  template< typename T1, typename T2, typename T3 >
164  struct IsEvaluationRequired {
165  enum : bool { value = ( evaluateLeft || evaluateRight ) };
166  };
168  //**********************************************************************************************
169 
170  //**********************************************************************************************
172 
175  template< typename T1, typename T2, typename T3 >
176  struct UseVectorizedKernel {
177  enum : bool { value = useOptimizedKernels &&
179  T1::simdEnabled && T3::simdEnabled &&
183  , ElementType_<T3> >::value &&
186  };
188  //**********************************************************************************************
189 
190  //**********************************************************************************************
192 
196  template< typename T1, typename T2, typename T3 >
197  struct UseOptimizedKernel {
198  enum : bool { value = useOptimizedKernels &&
199  !UseVectorizedKernel<T1,T2,T3>::value &&
201  !IsResizable< ElementType_<T1> >::value &&
203  };
205  //**********************************************************************************************
206 
207  //**********************************************************************************************
209 
212  template< typename T1, typename T2, typename T3 >
213  struct UseDefaultKernel {
214  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
215  !UseOptimizedKernel<T1,T2,T3>::value };
216  };
218  //**********************************************************************************************
219 
220  //**********************************************************************************************
222 
225  using ForwardFunctor = IfTrue_< HERM
226  , DeclHerm
227  , IfTrue_< SYM
228  , DeclSym
229  , IfTrue_< LOW
230  , IfTrue_< UPP
231  , DeclDiag
232  , DeclLow >
233  , IfTrue_< UPP
234  , DeclUpp
235  , Noop > > > >;
237  //**********************************************************************************************
238 
239  public:
240  //**Type definitions****************************************************************************
243 
249  using ReturnType = const ElementType;
250  using CompositeType = const ResultType;
251 
253  using LeftOperand = If_< IsExpression<MT1>, const MT1, const MT1& >;
254 
256  using RightOperand = If_< IsExpression<MT2>, const MT2, const MT2& >;
257 
260 
263  //**********************************************************************************************
264 
265  //**Compilation flags***************************************************************************
267  enum : bool { simdEnabled = !IsDiagonal<MT2>::value &&
268  MT2::simdEnabled &&
271 
273  enum : bool { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
274  !evaluateRight && MT2::smpAssignable };
275  //**********************************************************************************************
276 
277  //**SIMD properties*****************************************************************************
279  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
280  //**********************************************************************************************
281 
282  //**Constructor*********************************************************************************
288  explicit inline SMatDMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
289  : lhs_( lhs ) // Left-hand side sparse matrix of the multiplication expression
290  , rhs_( rhs ) // Right-hand side dense matrix of the multiplication expression
291  {
292  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
293  }
294  //**********************************************************************************************
295 
296  //**Access operator*****************************************************************************
303  inline ReturnType operator()( size_t i, size_t j ) const {
304  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
305  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
306 
307  if( IsDiagonal<MT1>::value ) {
308  return lhs_(i,i) * rhs_(i,j);
309  }
310  else if( IsDiagonal<MT2>::value ) {
311  return lhs_(i,j) * rhs_(j,j);
312  }
314  const size_t begin( ( IsUpper<MT1>::value )
315  ?( ( IsLower<MT2>::value )
316  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
317  , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
318  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
319  :( ( IsLower<MT2>::value )
320  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
321  :( 0UL ) ) );
322  const size_t end( ( IsLower<MT1>::value )
323  ?( ( IsUpper<MT2>::value )
324  ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
325  , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
326  :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
327  :( ( IsUpper<MT2>::value )
328  ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
329  :( lhs_.columns() ) ) );
330 
331  if( begin >= end ) return ElementType();
332 
333  const size_t n( end - begin );
334 
335  return subvector( row( lhs_, i, unchecked ), begin, n, unchecked ) *
336  subvector( column( rhs_, j, unchecked ), begin, n, unchecked );
337  }
338  else {
339  return row( lhs_, i, unchecked ) * column( rhs_, j, unchecked );
340  }
341  }
342  //**********************************************************************************************
343 
344  //**At function*********************************************************************************
352  inline ReturnType at( size_t i, size_t j ) const {
353  if( i >= lhs_.rows() ) {
354  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
355  }
356  if( j >= rhs_.columns() ) {
357  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
358  }
359  return (*this)(i,j);
360  }
361  //**********************************************************************************************
362 
363  //**Rows function*******************************************************************************
368  inline size_t rows() const noexcept {
369  return lhs_.rows();
370  }
371  //**********************************************************************************************
372 
373  //**Columns function****************************************************************************
378  inline size_t columns() const noexcept {
379  return rhs_.columns();
380  }
381  //**********************************************************************************************
382 
383  //**Left operand access*************************************************************************
388  inline LeftOperand leftOperand() const noexcept {
389  return lhs_;
390  }
391  //**********************************************************************************************
392 
393  //**Right operand access************************************************************************
398  inline RightOperand rightOperand() const noexcept {
399  return rhs_;
400  }
401  //**********************************************************************************************
402 
403  //**********************************************************************************************
409  template< typename T >
410  inline bool canAlias( const T* alias ) const noexcept {
411  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
412  }
413  //**********************************************************************************************
414 
415  //**********************************************************************************************
421  template< typename T >
422  inline bool isAliased( const T* alias ) const noexcept {
423  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
424  }
425  //**********************************************************************************************
426 
427  //**********************************************************************************************
432  inline bool isAligned() const noexcept {
433  return rhs_.isAligned();
434  }
435  //**********************************************************************************************
436 
437  //**********************************************************************************************
442  inline bool canSMPAssign() const noexcept {
443  return ( rows() * columns() >= SMP_SMATDMATMULT_THRESHOLD ) && !IsDiagonal<MT2>::value;
444  }
445  //**********************************************************************************************
446 
447  private:
448  //**Member variables****************************************************************************
451  //**********************************************************************************************
452 
453  //**Assignment to dense matrices****************************************************************
466  template< typename MT // Type of the target dense matrix
467  , bool SO > // Storage order of the target dense matrix
468  friend inline void assign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
469  {
471 
472  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
473  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
474 
475  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
476  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
477 
478  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
479  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
480  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
481  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
482  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
483  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
484 
485  SMatDMatMultExpr::selectAssignKernel( ~lhs, A, B );
486  }
488  //**********************************************************************************************
489 
490  //**Default assignment to dense matrices********************************************************
504  template< typename MT3 // Type of the left-hand side target matrix
505  , typename MT4 // Type of the left-hand side matrix operand
506  , typename MT5 > // Type of the right-hand side matrix operand
508  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
509  {
511 
512  const size_t block( Or< IsRowMajorMatrix<MT3>, IsDiagonal<MT5> >::value ? B.columns() : 64UL );
513 
514  reset( C );
515 
516  for( size_t jj=0UL; jj<B.columns(); jj+=block )
517  {
518  const size_t jtmp( min( jj+block, B.columns() ) );
519 
520  for( size_t i=0UL; i<A.rows(); ++i )
521  {
522  ConstIterator element( A.begin(i) );
523  const ConstIterator end( A.end(i) );
524 
525  for( ; element!=end; ++element )
526  {
527  const size_t i1( element->index() );
528 
530  {
531  C(i,i1) = element->value() * B(i1,i1);
532  }
533  else
534  {
535  const size_t jbegin( ( IsUpper<MT5>::value )
536  ?( ( UPP )
537  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
538  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
539  :( jj ) );
540  const size_t jend( ( IsLower<MT5>::value )
541  ?( ( SYM || HERM || LOW )
542  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
543  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
544  :( SYM || HERM || LOW ? min(i+1UL,jtmp) : jtmp ) );
545 
546  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
547  continue;
548 
549  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
550 
551  for( size_t j=jbegin; j<jend; ++j ) {
552  if( isDefault( C(i,j) ) )
553  C(i,j) = element->value() * B(i1,j);
554  else
555  C(i,j) += element->value() * B(i1,j);
556  }
557  }
558  }
559  }
560  }
561 
562  if( SYM || HERM ) {
563  for( size_t i=0UL; i<A.rows(); ++i ) {
564  for( size_t j=i+1UL; j<B.columns(); ++j ) {
565  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
566  }
567  }
568  }
569  }
571  //**********************************************************************************************
572 
573  //**Optimized assignment to dense matrices******************************************************
587  template< typename MT3 // Type of the left-hand side target matrix
588  , typename MT4 // Type of the left-hand side matrix operand
589  , typename MT5 > // Type of the right-hand side matrix operand
591  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
592  {
594 
595  const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 64UL );
596 
597  reset( C );
598 
599  for( size_t jj=0UL; jj<B.columns(); jj+=block )
600  {
601  const size_t jtmp( min( jj+block, B.columns() ) );
602 
603  for( size_t i=0UL; i<A.rows(); ++i )
604  {
605  const ConstIterator end( A.end(i) );
606  ConstIterator element( A.begin(i) );
607 
608  const size_t nonzeros( A.nonZeros(i) );
609  const size_t kpos( nonzeros & size_t(-4) );
610  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
611 
612  for( size_t k=0UL; k<kpos; k+=4UL )
613  {
614  const size_t i1( element->index() );
615  const ET1 v1( element->value() );
616  ++element;
617  const size_t i2( element->index() );
618  const ET1 v2( element->value() );
619  ++element;
620  const size_t i3( element->index() );
621  const ET1 v3( element->value() );
622  ++element;
623  const size_t i4( element->index() );
624  const ET1 v4( element->value() );
625  ++element;
626 
627  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
628 
629  const size_t jbegin( ( IsUpper<MT5>::value )
630  ?( ( UPP )
631  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
632  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
633  :( UPP ? max(i,jj) : jj ) );
634  const size_t jend( ( IsLower<MT5>::value )
635  ?( ( SYM || HERM || LOW )
636  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) )
637  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) ) )
638  :( SYM || HERM || LOW ? min(i+1UL,jtmp) : jtmp ) );
639 
640  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
641  continue;
642 
643  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
644 
645  const size_t jnum( jend - jbegin );
646  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
647  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
648 
649  for( size_t j=jbegin; j<jpos; j+=4UL ) {
650  C(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
651  C(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
652  C(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
653  C(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
654  }
655  for( size_t j=jpos; j<jend; ++j ) {
656  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
657  }
658  }
659 
660  for( ; element!=end; ++element )
661  {
662  const size_t i1( element->index() );
663  const ET1 v1( element->value() );
664 
665  const size_t jbegin( ( IsUpper<MT5>::value )
666  ?( ( UPP )
667  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
668  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
669  :( UPP ? max(i,jj) : jj ) );
670  const size_t jend( ( IsLower<MT5>::value )
671  ?( ( SYM || HERM || LOW )
672  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
673  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
674  :( SYM || HERM || LOW ? min(i+1UL,jtmp) : jtmp ) );
675 
676  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
677  continue;
678 
679  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
680 
681  const size_t jnum( jend - jbegin );
682  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
683  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
684 
685  for( size_t j=jbegin; j<jpos; j+=4UL ) {
686  C(i,j ) += v1 * B(i1,j );
687  C(i,j+1UL) += v1 * B(i1,j+1UL);
688  C(i,j+2UL) += v1 * B(i1,j+2UL);
689  C(i,j+3UL) += v1 * B(i1,j+3UL);
690  }
691  for( size_t j=jpos; j<jend; ++j ) {
692  C(i,j) += v1 * B(i1,j);
693  }
694  }
695  }
696  }
697 
698  if( SYM || HERM ) {
699  for( size_t i=0UL; i<A.rows(); ++i ) {
700  for( size_t j=i+1UL; j<B.columns(); ++j ) {
701  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
702  }
703  }
704  }
705  }
707  //**********************************************************************************************
708 
709  //**Vectorized assignment to dense matrices*****************************************************
723  template< typename MT3 // Type of the left-hand side target matrix
724  , typename MT4 // Type of the left-hand side matrix operand
725  , typename MT5 > // Type of the right-hand side matrix operand
727  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
728  {
730 
731  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT5>::value );
732 
733  reset( C );
734 
735  for( size_t i=0UL; i<A.rows(); ++i )
736  {
737  const ConstIterator end( A.end(i) );
738  ConstIterator element( A.begin(i) );
739 
740  const size_t nonzeros( A.nonZeros(i) );
741  const size_t kpos( nonzeros & size_t(-4) );
742  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
743 
744  for( size_t k=0UL; k<kpos; k+=4UL )
745  {
746  const size_t i1( element->index() );
747  const ET1 v1( element->value() );
748  ++element;
749  const size_t i2( element->index() );
750  const ET1 v2( element->value() );
751  ++element;
752  const size_t i3( element->index() );
753  const ET1 v3( element->value() );
754  ++element;
755  const size_t i4( element->index() );
756  const ET1 v4( element->value() );
757  ++element;
758 
759  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
760 
761  const SIMDType xmm1( set( v1 ) );
762  const SIMDType xmm2( set( v2 ) );
763  const SIMDType xmm3( set( v3 ) );
764  const SIMDType xmm4( set( v4 ) );
765 
766  const size_t jbegin( ( IsUpper<MT5>::value )
768  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
769  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
770  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
771  const size_t jend( ( IsLower<MT5>::value )
773  ?( SYM || HERM || LOW ? min(i+1UL,i4) : i4 )
774  :( SYM || HERM || LOW ? min(i,i4)+1UL : i4+1UL ) )
775  :( SYM || HERM || LOW ? i+1UL : B.columns() ) );
776  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
777 
778  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
779  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
780 
781  size_t j( jbegin );
782 
783  for( ; j<jpos; j+=SIMDSIZE ) {
784  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) + xmm2 * B.load(i2,j) + xmm3 * B.load(i3,j) + xmm4 * B.load(i4,j) );
785  }
786  for( ; remainder && j<jend; ++j ) {
787  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
788  }
789  }
790 
791  for( ; element!=end; ++element )
792  {
793  const size_t i1( element->index() );
794  const ET1 v1( element->value() );
795 
796  const SIMDType xmm1( set( v1 ) );
797 
798  const size_t jbegin( ( IsUpper<MT5>::value )
800  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
801  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
802  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
803  const size_t jend( ( IsLower<MT5>::value )
805  ?( SYM || HERM || LOW ? min(i+1UL,i1) : i1 )
806  :( SYM || HERM || LOW ? min(i,i1)+1UL : i1+1UL ) )
807  :( SYM || HERM || LOW ? i+1UL : B.columns() ) );
808  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
809 
810  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
811  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
812 
813  size_t j( jbegin );
814 
815  for( ; j<jpos; j+=SIMDSIZE ) {
816  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) );
817  }
818  for( ; remainder && j<jend; ++j ) {
819  C(i,j) += v1 * B(i1,j);
820  }
821  }
822  }
823 
824  if( SYM || HERM ) {
825  for( size_t i=0UL; i<A.rows(); ++i ) {
826  for( size_t j=i+1UL; j<B.columns(); ++j ) {
827  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
828  }
829  }
830  }
831  }
833  //**********************************************************************************************
834 
835  //**Assignment to sparse matrices***************************************************************
848  template< typename MT // Type of the target sparse matrix
849  , bool SO > // Storage order of the target sparse matrix
850  friend inline void assign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
851  {
853 
855 
862 
863  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
864  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
865 
866  const ForwardFunctor fwd;
867 
868  const TmpType tmp( serial( rhs ) );
869  assign( ~lhs, fwd( tmp ) );
870  }
872  //**********************************************************************************************
873 
874  //**Addition assignment to dense matrices*******************************************************
887  template< typename MT // Type of the target dense matrix
888  , bool SO > // Storage order of the target dense matrix
889  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
890  {
892 
893  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
894  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
895 
896  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
897  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
898 
899  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
900  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
901  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
902  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
903  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
904  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
905 
906  SMatDMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
907  }
909  //**********************************************************************************************
910 
911  //**Default addition assignment to dense matrices***********************************************
925  template< typename MT3 // Type of the left-hand side target matrix
926  , typename MT4 // Type of the left-hand side matrix operand
927  , typename MT5 > // Type of the right-hand side matrix operand
929  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
930  {
932 
933  const size_t block( Or< IsRowMajorMatrix<MT3>, IsDiagonal<MT5> >::value ? B.columns() : 64UL );
934 
935  for( size_t jj=0UL; jj<B.columns(); jj+=block )
936  {
937  const size_t jtmp( min( jj+block, B.columns() ) );
938 
939  for( size_t i=0UL; i<A.rows(); ++i )
940  {
941  const ConstIterator end( A.end(i) );
942  ConstIterator element( A.begin(i) );
943 
944  for( ; element!=end; ++element )
945  {
946  const size_t i1( element->index() );
947 
949  {
950  C(i,i1) += element->value() * B(i1,i1);
951  }
952  else
953  {
954  const size_t jbegin( ( IsUpper<MT5>::value )
955  ?( ( UPP )
956  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
957  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
958  :( jj ) );
959  const size_t jend( ( IsLower<MT5>::value )
960  ?( ( LOW )
961  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
962  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
963  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
964 
965  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
966  continue;
967 
968  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
969 
970  const size_t jnum( jend - jbegin );
971  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
972  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
973 
974  for( size_t j=jbegin; j<jpos; j+=4UL ) {
975  C(i,j ) += element->value() * B(i1,j );
976  C(i,j+1UL) += element->value() * B(i1,j+1UL);
977  C(i,j+2UL) += element->value() * B(i1,j+2UL);
978  C(i,j+3UL) += element->value() * B(i1,j+3UL);
979  }
980  for( size_t j=jpos; j<jend; ++j ) {
981  C(i,j) += element->value() * B(i1,j);
982  }
983  }
984  }
985  }
986  }
987  }
989  //**********************************************************************************************
990 
991  //**Optimized addition assignment to dense matrices*********************************************
1005  template< typename MT3 // Type of the left-hand side target matrix
1006  , typename MT4 // Type of the left-hand side matrix operand
1007  , typename MT5 > // Type of the right-hand side matrix operand
1009  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1010  {
1012 
1013  const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 64UL );
1014 
1015  for( size_t jj=0UL; jj<B.columns(); jj+=block )
1016  {
1017  const size_t jtmp( min( jj+block, B.columns() ) );
1018 
1019  for( size_t i=0UL; i<A.rows(); ++i )
1020  {
1021  const ConstIterator end( A.end(i) );
1022  ConstIterator element( A.begin(i) );
1023 
1024  const size_t nonzeros( A.nonZeros(i) );
1025  const size_t kpos( nonzeros & size_t(-4) );
1026  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1027 
1028  for( size_t k=0UL; k<kpos; k+=4UL )
1029  {
1030  const size_t i1( element->index() );
1031  const ET1 v1( element->value() );
1032  ++element;
1033  const size_t i2( element->index() );
1034  const ET1 v2( element->value() );
1035  ++element;
1036  const size_t i3( element->index() );
1037  const ET1 v3( element->value() );
1038  ++element;
1039  const size_t i4( element->index() );
1040  const ET1 v4( element->value() );
1041  ++element;
1042 
1043  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1044 
1045  const size_t jbegin( ( IsUpper<MT5>::value )
1046  ?( ( UPP )
1047  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1048  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1049  :( UPP ? max(i,jj) : jj ) );
1050  const size_t jend( ( IsLower<MT5>::value )
1051  ?( ( LOW )
1052  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) )
1053  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) ) )
1054  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1055 
1056  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1057  continue;
1058 
1059  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1060 
1061  const size_t jnum( jend - jbegin );
1062  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1063  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1064 
1065  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1066  C(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1067  C(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1068  C(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1069  C(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1070  }
1071  for( size_t j=jpos; j<jend; ++j ) {
1072  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1073  }
1074  }
1075 
1076  for( ; element!=end; ++element )
1077  {
1078  const size_t i1( element->index() );
1079  const ET1 v1( element->value() );
1080 
1081  const size_t jbegin( ( IsUpper<MT5>::value )
1082  ?( ( UPP )
1083  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1084  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1085  :( UPP ? max(i,jj) : jj ) );
1086  const size_t jend( ( IsLower<MT5>::value )
1087  ?( ( LOW )
1088  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
1089  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
1090  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1091 
1092  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1093  continue;
1094 
1095  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1096 
1097  const size_t jnum( jend - jbegin );
1098  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1099  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1100 
1101  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1102  C(i,j ) += v1 * B(i1,j );
1103  C(i,j+1UL) += v1 * B(i1,j+1UL);
1104  C(i,j+2UL) += v1 * B(i1,j+2UL);
1105  C(i,j+3UL) += v1 * B(i1,j+3UL);
1106  }
1107  for( size_t j=jpos; j<jend; ++j ) {
1108  C(i,j) += v1 * B(i1,j);
1109  }
1110  }
1111  }
1112  }
1113  }
1115  //**********************************************************************************************
1116 
1117  //**Vectorized addition assignment to dense matrices********************************************
1131  template< typename MT3 // Type of the left-hand side target matrix
1132  , typename MT4 // Type of the left-hand side matrix operand
1133  , typename MT5 > // Type of the right-hand side matrix operand
1135  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1136  {
1138 
1139  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT5>::value );
1140 
1141  for( size_t i=0UL; i<A.rows(); ++i )
1142  {
1143  const ConstIterator end( A.end(i) );
1144  ConstIterator element( A.begin(i) );
1145 
1146  const size_t nonzeros( A.nonZeros(i) );
1147  const size_t kpos( nonzeros & size_t(-4) );
1148  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1149 
1150  for( size_t k=0UL; k<kpos; k+=4UL )
1151  {
1152  const size_t i1( element->index() );
1153  const ET1 v1( element->value() );
1154  ++element;
1155  const size_t i2( element->index() );
1156  const ET1 v2( element->value() );
1157  ++element;
1158  const size_t i3( element->index() );
1159  const ET1 v3( element->value() );
1160  ++element;
1161  const size_t i4( element->index() );
1162  const ET1 v4( element->value() );
1163  ++element;
1164 
1165  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1166 
1167  const SIMDType xmm1( set( v1 ) );
1168  const SIMDType xmm2( set( v2 ) );
1169  const SIMDType xmm3( set( v3 ) );
1170  const SIMDType xmm4( set( v4 ) );
1171 
1172  const size_t jbegin( ( IsUpper<MT5>::value )
1174  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
1175  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
1176  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
1177  const size_t jend( ( IsLower<MT5>::value )
1179  ?( LOW ? min(i+1UL,i4) : i4 )
1180  :( LOW ? min(i,i4)+1UL : i4+1UL ) )
1181  :( LOW ? i+1UL : B.columns() ) );
1182  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1183 
1184  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1185  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
1186 
1187  size_t j( jbegin );
1188 
1189  for( ; j<jpos; j+=SIMDSIZE ) {
1190  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) + xmm2 * B.load(i2,j) + xmm3 * B.load(i3,j) + xmm4 * B.load(i4,j) );
1191  }
1192  for( ; remainder && j<jend; ++j ) {
1193  C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1194  }
1195  }
1196 
1197  for( ; element!=end; ++element )
1198  {
1199  const size_t i1( element->index() );
1200  const ET1 v1( element->value() );
1201 
1202  const SIMDType xmm1( set( v1 ) );
1203 
1204  const size_t jbegin( ( IsUpper<MT5>::value )
1206  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
1207  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
1208  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
1209  const size_t jend( ( IsLower<MT5>::value )
1211  ?( LOW ? min(i+1UL,i1) : i1 )
1212  :( LOW ? min(i,i1)+1UL : i1+1UL ) )
1213  :( LOW ? i+1UL : B.columns() ) );
1214  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1215 
1216  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1217  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
1218 
1219  size_t j( jbegin );
1220 
1221  for( ; j<jpos; j+=SIMDSIZE ) {
1222  C.store( i, j, C.load(i,j) + xmm1 * B.load(i1,j) );
1223  }
1224  for( ; remainder && j<jend; ++j ) {
1225  C(i,j) += v1 * B(i1,j);
1226  }
1227  }
1228  }
1229  }
1231  //**********************************************************************************************
1232 
1233  //**Addition assignment to sparse matrices******************************************************
1234  // No special implementation for the addition assignment to sparse matrices.
1235  //**********************************************************************************************
1236 
1237  //**Subtraction assignment to dense matrices****************************************************
1250  template< typename MT // Type of the target dense matrix
1251  , bool SO > // Storage order of the target dense matrix
1252  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1253  {
1255 
1256  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1257  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1258 
1259  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
1260  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
1261 
1262  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1263  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1264  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1265  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1266  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1267  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1268 
1269  SMatDMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1270  }
1272  //**********************************************************************************************
1273 
1274  //**Default subtraction assignment to dense matrices********************************************
1288  template< typename MT3 // Type of the left-hand side target matrix
1289  , typename MT4 // Type of the left-hand side matrix operand
1290  , typename MT5 > // Type of the right-hand side matrix operand
1292  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1293  {
1295 
1296  const size_t block( Or< IsRowMajorMatrix<MT3>, IsDiagonal<MT5> >::value ? B.columns() : 64UL );
1297 
1298  for( size_t jj=0UL; jj<B.columns(); jj+=block )
1299  {
1300  const size_t jtmp( min( jj+block, B.columns() ) );
1301 
1302  for( size_t i=0UL; i<A.rows(); ++i )
1303  {
1304  const ConstIterator end( A.end(i) );
1305  ConstIterator element( A.begin(i) );
1306 
1307  for( ; element!=end; ++element )
1308  {
1309  const size_t i1( element->index() );
1310 
1312  {
1313  C(i,i1) -= element->value() * B(i1,i1);
1314  }
1315  else
1316  {
1317  const size_t jbegin( ( IsUpper<MT5>::value )
1318  ?( ( UPP )
1319  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1320  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1321  :( jj ) );
1322  const size_t jend( ( IsLower<MT5>::value )
1323  ?( ( LOW )
1324  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
1325  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
1326  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1327 
1328  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1329  continue;
1330 
1331  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1332 
1333  const size_t jnum( jend - jbegin );
1334  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1335  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1336 
1337  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1338  C(i,j ) -= element->value() * B(i1,j );
1339  C(i,j+1UL) -= element->value() * B(i1,j+1UL);
1340  C(i,j+2UL) -= element->value() * B(i1,j+2UL);
1341  C(i,j+3UL) -= element->value() * B(i1,j+3UL);
1342  }
1343  for( size_t j=jpos; j<jend; ++j ) {
1344  C(i,j) -= element->value() * B(i1,j);
1345  }
1346  }
1347  }
1348  }
1349  }
1350  }
1352  //**********************************************************************************************
1353 
1354  //**Optimized subtraction assignment to dense matrices******************************************
1368  template< typename MT3 // Type of the left-hand side target matrix
1369  , typename MT4 // Type of the left-hand side matrix operand
1370  , typename MT5 > // Type of the right-hand side matrix operand
1372  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1373  {
1375 
1376  const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 64UL );
1377 
1378  for( size_t jj=0UL; jj<B.columns(); jj+=block )
1379  {
1380  const size_t jtmp( min( jj+block, B.columns() ) );
1381 
1382  for( size_t i=0UL; i<A.rows(); ++i )
1383  {
1384  const ConstIterator end( A.end(i) );
1385  ConstIterator element( A.begin(i) );
1386 
1387  const size_t nonzeros( A.nonZeros(i) );
1388  const size_t kpos( nonzeros & size_t(-4) );
1389  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1390 
1391  for( size_t k=0UL; k<kpos; k+=4UL )
1392  {
1393  const size_t i1( element->index() );
1394  const ET1 v1( element->value() );
1395  ++element;
1396  const size_t i2( element->index() );
1397  const ET1 v2( element->value() );
1398  ++element;
1399  const size_t i3( element->index() );
1400  const ET1 v3( element->value() );
1401  ++element;
1402  const size_t i4( element->index() );
1403  const ET1 v4( element->value() );
1404  ++element;
1405 
1406  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1407 
1408  const size_t jbegin( ( IsUpper<MT5>::value )
1409  ?( ( UPP )
1410  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1411  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1412  :( UPP ? max(i,jj) : jj ) );
1413  const size_t jend( ( IsLower<MT5>::value )
1414  ?( ( LOW )
1415  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) )
1416  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i4 : i4+1UL ) ) ) )
1417  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1418 
1419  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1420  continue;
1421 
1422  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1423 
1424  const size_t jnum( jend - jbegin );
1425  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1426  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1427 
1428  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1429  C(i,j ) -= v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1430  C(i,j+1UL) -= v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1431  C(i,j+2UL) -= v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1432  C(i,j+3UL) -= v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1433  }
1434  for( size_t j=jpos; j<jend; ++j ) {
1435  C(i,j) -= v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1436  }
1437  }
1438 
1439  for( ; element!=end; ++element )
1440  {
1441  const size_t i1( element->index() );
1442  const ET1 v1( element->value() );
1443 
1444  const size_t jbegin( ( IsUpper<MT5>::value )
1445  ?( ( UPP )
1446  ?( max( i, jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) )
1447  :( max( jj, ( IsStrictlyUpper<MT5>::value ? i1+1UL : i1 ) ) ) )
1448  :( UPP ? max(i,jj) : jj ) );
1449  const size_t jend( ( IsLower<MT5>::value )
1450  ?( ( LOW )
1451  ?( min( i+1UL, jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) )
1452  :( min( jtmp, ( IsStrictlyLower<MT5>::value ? i1 : i1+1UL ) ) ) )
1453  :( LOW ? min(i+1UL,jtmp) : jtmp ) );
1454 
1455  if( ( LOW || UPP || IsTriangular<MT5>::value ) && ( jbegin >= jend ) )
1456  continue;
1457 
1458  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1459 
1460  const size_t jnum( jend - jbegin );
1461  const size_t jpos( jbegin + ( jnum & size_t(-4) ) );
1462  BLAZE_INTERNAL_ASSERT( ( jbegin + jnum - ( jnum % 4UL ) ) == jpos, "Invalid end calculation" );
1463 
1464  for( size_t j=jbegin; j<jpos; j+=4UL ) {
1465  C(i,j ) -= v1 * B(i1,j );
1466  C(i,j+1UL) -= v1 * B(i1,j+1UL);
1467  C(i,j+2UL) -= v1 * B(i1,j+2UL);
1468  C(i,j+3UL) -= v1 * B(i1,j+3UL);
1469  }
1470  for( size_t j=jpos; j<jend; ++j ) {
1471  C(i,j) -= v1 * B(i1,j);
1472  }
1473  }
1474  }
1475  }
1476  }
1478  //**********************************************************************************************
1479 
1480  //**Vectorized subtraction assignment to dense matrices*****************************************
1494  template< typename MT3 // Type of the left-hand side target matrix
1495  , typename MT4 // Type of the left-hand side matrix operand
1496  , typename MT5 > // Type of the right-hand side matrix operand
1498  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1499  {
1501 
1502  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT5>::value );
1503 
1504  for( size_t i=0UL; i<A.rows(); ++i )
1505  {
1506  const ConstIterator end( A.end(i) );
1507  ConstIterator element( A.begin(i) );
1508 
1509  const size_t nonzeros( A.nonZeros(i) );
1510  const size_t kpos( nonzeros & size_t(-4) );
1511  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1512 
1513  for( size_t k=0UL; k<kpos; k+=4UL )
1514  {
1515  const size_t i1( element->index() );
1516  const ET1 v1( element->value() );
1517  ++element;
1518  const size_t i2( element->index() );
1519  const ET1 v2( element->value() );
1520  ++element;
1521  const size_t i3( element->index() );
1522  const ET1 v3( element->value() );
1523  ++element;
1524  const size_t i4( element->index() );
1525  const ET1 v4( element->value() );
1526  ++element;
1527 
1528  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1529 
1530  const SIMDType xmm1( set( v1 ) );
1531  const SIMDType xmm2( set( v2 ) );
1532  const SIMDType xmm3( set( v3 ) );
1533  const SIMDType xmm4( set( v4 ) );
1534 
1535  const size_t jbegin( ( IsUpper<MT5>::value )
1537  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
1538  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
1539  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
1540  const size_t jend( ( IsLower<MT5>::value )
1542  ?( LOW ? min(i+1UL,i4) : i4 )
1543  :( LOW ? min(i,i4)+1UL : i4+1UL ) )
1544  :( LOW ? i+1UL : B.columns() ) );
1545  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1546 
1547  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1548  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
1549 
1550  size_t j( jbegin );
1551 
1552  for( ; j<jpos; j+=SIMDSIZE ) {
1553  C.store( i, j, C.load(i,j) - xmm1 * B.load(i1,j) - xmm2 * B.load(i2,j) - xmm3 * B.load(i3,j) - xmm4 * B.load(i4,j) );
1554  }
1555  for( ; remainder && j<jend; ++j ) {
1556  C(i,j) -= v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1557  }
1558  }
1559 
1560  for( ; element!=end; ++element )
1561  {
1562  const size_t i1( element->index() );
1563  const ET1 v1( element->value() );
1564 
1565  const SIMDType xmm1( set( v1 ) );
1566 
1567  const size_t jbegin( ( IsUpper<MT5>::value )
1569  ?( ( UPP ? max(i,i1+1UL) : i1+1UL ) & size_t(-SIMDSIZE) )
1570  :( ( UPP ? max(i,i1) : i1 ) & size_t(-SIMDSIZE) ) )
1571  :( UPP ? ( i & size_t(-SIMDSIZE) ) : 0UL ) );
1572  const size_t jend( ( IsLower<MT5>::value )
1574  ?( LOW ? min(i+1UL,i1) : i1 )
1575  :( LOW ? min(i,i1)+1UL : i1+1UL ) )
1576  :( LOW ? i+1UL : B.columns() ) );
1577  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1578 
1579  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1580  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
1581 
1582  size_t j( jbegin );
1583 
1584  for( ; j<jpos; j+=SIMDSIZE ) {
1585  C.store( i, j, C.load(i,j) - xmm1 * B.load(i1,j) );
1586  }
1587  for( ; remainder && j<jend; ++j ) {
1588  C(i,j) -= v1 * B(i1,j);
1589  }
1590  }
1591  }
1592  }
1594  //**********************************************************************************************
1595 
1596  //**Subtraction assignment to sparse matrices***************************************************
1597  // No special implementation for the subtraction assignment to sparse matrices.
1598  //**********************************************************************************************
1599 
1600  //**Schur product assignment to dense matrices**************************************************
1613  template< typename MT // Type of the target dense matrix
1614  , bool SO > // Storage order of the target dense matrix
1615  friend inline void schurAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1616  {
1618 
1622 
1623  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1624  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1625 
1626  const ResultType tmp( serial( rhs ) );
1627  schurAssign( ~lhs, tmp );
1628  }
1630  //**********************************************************************************************
1631 
1632  //**Schur product assignment to sparse matrices*************************************************
1633  // No special implementation for the Schur product assignment to sparse matrices.
1634  //**********************************************************************************************
1635 
1636  //**Multiplication assignment to dense matrices*************************************************
1637  // No special implementation for the multiplication assignment to dense matrices.
1638  //**********************************************************************************************
1639 
1640  //**Multiplication assignment to sparse matrices************************************************
1641  // No special implementation for the multiplication assignment to sparse matrices.
1642  //**********************************************************************************************
1643 
1644  //**SMP assignment to dense matrices************************************************************
1659  template< typename MT // Type of the target dense matrix
1660  , bool SO > // Storage order of the target dense matrix
1662  smpAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1663  {
1665 
1666  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1667  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1668 
1669  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1670  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1671 
1672  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1673  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1674  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1675  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1676  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1677  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1678 
1679  smpAssign( ~lhs, A * B );
1680  }
1682  //**********************************************************************************************
1683 
1684  //**SMP assignment to sparse matrices***********************************************************
1699  template< typename MT // Type of the target sparse matrix
1700  , bool SO > // Storage order of the target sparse matrix
1702  smpAssign( SparseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1703  {
1705 
1707 
1714 
1715  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1716  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1717 
1718  const ForwardFunctor fwd;
1719 
1720  const TmpType tmp( rhs );
1721  smpAssign( ~lhs, fwd( tmp ) );
1722  }
1724  //**********************************************************************************************
1725 
1726  //**SMP addition assignment to dense matrices***************************************************
1742  template< typename MT // Type of the target dense matrix
1743  , bool SO > // Storage order of the target dense matrix
1746  {
1748 
1749  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1750  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1751 
1752  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1753  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1754 
1755  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1756  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1757  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1758  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1759  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1760  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1761 
1762  smpAddAssign( ~lhs, A * B );
1763  }
1765  //**********************************************************************************************
1766 
1767  //**SMP addition assignment to sparse matrices**************************************************
1768  // No special implementation for the SMP addition assignment to sparse matrices.
1769  //**********************************************************************************************
1770 
1771  //**SMP subtraction assignment to dense matrices************************************************
1787  template< typename MT // Type of the target dense matrix
1788  , bool SO > // Storage order of the target dense matrix
1791  {
1793 
1794  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1795  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1796 
1797  LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1798  RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1799 
1800  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1801  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1802  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1803  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1804  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1805  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1806 
1807  smpSubAssign( ~lhs, A * B );
1808  }
1810  //**********************************************************************************************
1811 
1812  //**SMP subtraction assignment to sparse matrices***********************************************
1813  // No special implementation for the SMP subtraction assignment to sparse matrices.
1814  //**********************************************************************************************
1815 
1816  //**SMP Schur product assignment to dense matrices**********************************************
1829  template< typename MT // Type of the target dense matrix
1830  , bool SO > // Storage order of the target dense matrix
1831  friend inline void smpSchurAssign( DenseMatrix<MT,SO>& lhs, const SMatDMatMultExpr& rhs )
1832  {
1834 
1838 
1839  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1840  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1841 
1842  const ResultType tmp( rhs );
1843  smpSchurAssign( ~lhs, tmp );
1844  }
1846  //**********************************************************************************************
1847 
1848  //**SMP Schur product assignment to sparse matrices*********************************************
1849  // No special implementation for the SMP Schur product assignment to sparse matrices.
1850  //**********************************************************************************************
1851 
1852  //**SMP multiplication assignment to dense matrices*********************************************
1853  // No special implementation for the SMP multiplication assignment to dense matrices.
1854  //**********************************************************************************************
1855 
1856  //**SMP multiplication assignment to sparse matrices********************************************
1857  // No special implementation for the SMP multiplication assignment to sparse matrices.
1858  //**********************************************************************************************
1859 
1860  //**Compile time checks*************************************************************************
1868  //**********************************************************************************************
1869 };
1870 //*************************************************************************************************
1871 
1872 
1873 
1874 
1875 //=================================================================================================
1876 //
1877 // GLOBAL BINARY ARITHMETIC OPERATORS
1878 //
1879 //=================================================================================================
1880 
1881 //*************************************************************************************************
1910 template< typename MT1 // Type of the left-hand side sparse matrix
1911  , typename MT2 > // Type of the right-hand side dense matrix
1912 inline decltype(auto)
1913  operator*( const SparseMatrix<MT1,false>& lhs, const DenseMatrix<MT2,false>& rhs )
1914 {
1916 
1917  if( (~lhs).columns() != (~rhs).rows() ) {
1918  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1919  }
1920 
1922  return ReturnType( ~lhs, ~rhs );
1923 }
1924 //*************************************************************************************************
1925 
1926 
1927 
1928 
1929 //=================================================================================================
1930 //
1931 // GLOBAL FUNCTIONS
1932 //
1933 //=================================================================================================
1934 
1935 //*************************************************************************************************
1959 template< typename MT1 // Type of the left-hand side sparse matrix
1960  , typename MT2 // Type of the right-hand side dense matrix
1961  , bool SF // Symmetry flag
1962  , bool HF // Hermitian flag
1963  , bool LF // Lower flag
1964  , bool UF > // Upper flag
1965 inline decltype(auto) declsym( const SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
1966 {
1968 
1969  if( !isSquare( dm ) ) {
1970  BLAZE_THROW_INVALID_ARGUMENT( "Invalid symmetric matrix specification" );
1971  }
1972 
1974  return ReturnType( dm.leftOperand(), dm.rightOperand() );
1975 }
1977 //*************************************************************************************************
1978 
1979 
1980 //*************************************************************************************************
2004 template< typename MT1 // Type of the left-hand side sparse matrix
2005  , typename MT2 // Type of the right-hand side dense matrix
2006  , bool SF // Symmetry flag
2007  , bool HF // Hermitian flag
2008  , bool LF // Lower flag
2009  , bool UF > // Upper flag
2010 inline decltype(auto) declherm( const SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2011 {
2013 
2014  if( !isSquare( dm ) ) {
2015  BLAZE_THROW_INVALID_ARGUMENT( "Invalid Hermitian matrix specification" );
2016  }
2017 
2019  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2020 }
2022 //*************************************************************************************************
2023 
2024 
2025 //*************************************************************************************************
2049 template< typename MT1 // Type of the left-hand side dense matrix
2050  , typename MT2 // Type of the right-hand side dense matrix
2051  , bool SF // Symmetry flag
2052  , bool HF // Hermitian flag
2053  , bool LF // Lower flag
2054  , bool UF > // Upper flag
2055 inline decltype(auto) decllow( const SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2056 {
2058 
2059  if( !isSquare( dm ) ) {
2060  BLAZE_THROW_INVALID_ARGUMENT( "Invalid lower matrix specification" );
2061  }
2062 
2064  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2065 }
2067 //*************************************************************************************************
2068 
2069 
2070 //*************************************************************************************************
2094 template< typename MT1 // Type of the left-hand side dense matrix
2095  , typename MT2 // Type of the right-hand side dense matrix
2096  , bool SF // Symmetry flag
2097  , bool HF // Hermitian flag
2098  , bool LF // Lower flag
2099  , bool UF > // Upper flag
2100 inline decltype(auto) declupp( const SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2101 {
2103 
2104  if( !isSquare( dm ) ) {
2105  BLAZE_THROW_INVALID_ARGUMENT( "Invalid upper matrix specification" );
2106  }
2107 
2109  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2110 }
2112 //*************************************************************************************************
2113 
2114 
2115 //*************************************************************************************************
2139 template< typename MT1 // Type of the left-hand side dense matrix
2140  , typename MT2 // Type of the right-hand side dense matrix
2141  , bool SF // Symmetry flag
2142  , bool HF // Hermitian flag
2143  , bool LF // Lower flag
2144  , bool UF > // Upper flag
2145 inline decltype(auto) decldiag( const SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2146 {
2148 
2149  if( !isSquare( dm ) ) {
2150  BLAZE_THROW_INVALID_ARGUMENT( "Invalid diagonal matrix specification" );
2151  }
2152 
2154  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2155 }
2157 //*************************************************************************************************
2158 
2159 
2160 
2161 
2162 //=================================================================================================
2163 //
2164 // SIZE SPECIALIZATIONS
2165 //
2166 //=================================================================================================
2167 
2168 //*************************************************************************************************
2170 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2171 struct Size< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 0UL >
2172  : public Size<MT1,0UL>
2173 {};
2174 
2175 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2176 struct Size< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 1UL >
2177  : public Size<MT2,1UL>
2178 {};
2180 //*************************************************************************************************
2181 
2182 
2183 
2184 
2185 //=================================================================================================
2186 //
2187 // ISALIGNED SPECIALIZATIONS
2188 //
2189 //=================================================================================================
2190 
2191 //*************************************************************************************************
2193 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2194 struct IsAligned< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2195  : public IsAligned<MT2>
2196 {};
2198 //*************************************************************************************************
2199 
2200 
2201 
2202 
2203 //=================================================================================================
2204 //
2205 // ISSYMMETRIC SPECIALIZATIONS
2206 //
2207 //=================================================================================================
2208 
2209 //*************************************************************************************************
2211 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2212 struct IsSymmetric< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2213  : public Or< Bool<SF>
2214  , And< Bool<HF>
2215  , IsBuiltin< ElementType_< SMatDMatMultExpr<MT1,MT2,false,true,false,false> > > >
2216  , And< Bool<LF>, Bool<UF> > >
2217 {};
2219 //*************************************************************************************************
2220 
2221 
2222 
2223 
2224 //=================================================================================================
2225 //
2226 // ISHERMITIAN SPECIALIZATIONS
2227 //
2228 //=================================================================================================
2229 
2230 //*************************************************************************************************
2232 template< typename MT1, typename MT2, bool SF, bool LF, bool UF >
2233 struct IsHermitian< SMatDMatMultExpr<MT1,MT2,SF,true,LF,UF> >
2234  : public TrueType
2235 {};
2237 //*************************************************************************************************
2238 
2239 
2240 
2241 
2242 //=================================================================================================
2243 //
2244 // ISLOWER SPECIALIZATIONS
2245 //
2246 //=================================================================================================
2247 
2248 //*************************************************************************************************
2250 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2251 struct IsLower< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2252  : public Or< Bool<LF>
2253  , And< IsLower<MT1>, IsLower<MT2> >
2254  , And< Or< Bool<SF>, Bool<HF> >
2255  , IsUpper<MT1>, IsUpper<MT2> > >
2256 {};
2258 //*************************************************************************************************
2259 
2260 
2261 
2262 
2263 //=================================================================================================
2264 //
2265 // ISUNILOWER SPECIALIZATIONS
2266 //
2267 //=================================================================================================
2268 
2269 //*************************************************************************************************
2271 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2272 struct IsUniLower< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2273  : public Or< And< IsUniLower<MT1>, IsUniLower<MT2> >
2274  , And< Or< Bool<SF>, Bool<HF> >
2275  , IsUniUpper<MT1>, IsUniUpper<MT2> > >
2276 {};
2278 //*************************************************************************************************
2279 
2280 
2281 
2282 
2283 //=================================================================================================
2284 //
2285 // ISSTRICTLYLOWER SPECIALIZATIONS
2286 //
2287 //=================================================================================================
2288 
2289 //*************************************************************************************************
2291 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2292 struct IsStrictlyLower< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2293  : public Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2294  , And< IsStrictlyLower<MT2>, IsLower<MT1> >
2295  , And< Or< Bool<SF>, Bool<HF> >
2296  , Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2297  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > > > >
2298 {};
2300 //*************************************************************************************************
2301 
2302 
2303 
2304 
2305 //=================================================================================================
2306 //
2307 // ISUPPER SPECIALIZATIONS
2308 //
2309 //=================================================================================================
2310 
2311 //*************************************************************************************************
2313 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2314 struct IsUpper< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2315  : public Or< Bool<UF>
2316  , And< IsUpper<MT1>, IsUpper<MT2> >
2317  , And< Or< Bool<SF>, Bool<HF> >
2318  , IsLower<MT1>, IsLower<MT2> > >
2319 {};
2321 //*************************************************************************************************
2322 
2323 
2324 
2325 
2326 //=================================================================================================
2327 //
2328 // ISUNIUPPER SPECIALIZATIONS
2329 //
2330 //=================================================================================================
2331 
2332 //*************************************************************************************************
2334 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2335 struct IsUniUpper< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2336  : public Or< And< IsUniUpper<MT1>, IsUniUpper<MT2> >
2337  , And< Or< Bool<SF>, Bool<HF> >
2338  , IsUniLower<MT1>, IsUniLower<MT2> > >
2339 {};
2341 //*************************************************************************************************
2342 
2343 
2344 
2345 
2346 //=================================================================================================
2347 //
2348 // ISSTRICTLYUPPER SPECIALIZATIONS
2349 //
2350 //=================================================================================================
2351 
2352 //*************************************************************************************************
2354 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2355 struct IsStrictlyUpper< SMatDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2356  : public Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2357  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> >
2358  , And< Or< Bool<SF>, Bool<HF> >
2359  , Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2360  , And< IsStrictlyLower<MT2>, IsLower<MT1> > > > >
2361 {};
2363 //*************************************************************************************************
2364 
2365 } // namespace blaze
2366 
2367 #endif
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:329
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: SMatDMatMultExpr.h:442
Header file for auxiliary alias declarations.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:131
Headerfile for the generic min algorithm.
Header file for the blaze::checked and blaze::unchecked instances.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:71
decltype(auto) decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as diagonal.
Definition: DMatDeclDiagExpr.h:996
Header file for the IsUniUpper type trait.
EnableIf_< IsDenseMatrix< MT1 > > smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:196
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:86
Header file for basic type definitions.
CompositeType_< MT2 > CT2
Composite type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:134
ElementType_< RT2 > ET2
Element type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:132
EnableIf_< IsDenseMatrix< MT1 > > smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:164
Header file for the serial shim.
Header file for the IsDiagonal type trait.
Generic wrapper for a compile time constant integral value.The IntegralConstant class template repres...
Definition: IntegralConstant.h:71
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the DeclUpp functor.
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:364
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:172
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:316
SMatDMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the SMatDMatMultExpr class.
Definition: SMatDMatMultExpr.h:288
ResultType_< MT1 > RT1
Result type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:129
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:588
constexpr Unchecked unchecked
Global Unchecked instance.The blaze::unchecked instance is an optional token for the creation of view...
Definition: Check.h:138
Header file for the And class template.
const ElementType_< MT > min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1903
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:87
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:171
RightOperand rightOperand() const noexcept
Returns the right-hand side dense matrix operand.
Definition: SMatDMatMultExpr.h:398
decltype(auto) declupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as upper.
Definition: DMatDeclUppExpr.h:1026
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:291
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Expression object for sparse matrix-dense matrix multiplications.The SMatDMatMultExpr class represent...
Definition: Forward.h:107
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:87
ElementType_< ResultType > ElementType
Resulting element type.
Definition: SMatDMatMultExpr.h:247
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
IfTrue_< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: SMatDMatMultExpr.h:262
Compile time check for data types.This type trait tests whether or not the given types can be combine...
Definition: IsSIMDCombinable.h:120
Header file for the IsUniLower type trait.
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:343
const ElementType_< MT > max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1950
EnableIf_< IsDenseMatrix< MT1 > > smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:133
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:80
Flag for Hermitian matrices.
Definition: SMatDMatMultExpr.h:151
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:129
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
Compile time check for the alignment of data types.This type trait tests whether the given data type ...
Definition: IsAligned.h:87
Constraint on the data type.
Constraint on the data type.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:71
LeftOperand leftOperand() const noexcept
Returns the left-hand side sparse matrix operand.
Definition: SMatDMatMultExpr.h:388
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: SMatDMatMultExpr.h:410
Compile time check for upper unitriangular matrices.This type trait tests whether or not the given te...
Definition: IsUniUpper.h:86
Headerfile for the generic max algorithm.
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:58
Header file for the DeclLow functor.
ElementType_< RT1 > ET1
Element type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:131
OppositeType_< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: SMatDMatMultExpr.h:245
Header file for the If class template.
Compile time check for row-major matrix types.This type trait tests whether or not the given template...
Definition: IsRowMajorMatrix.h:110
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
ResultType_< MT2 > RT2
Result type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:130
Generic wrapper for the decllow() function.
Definition: DeclLow.h:58
Compile time check for data types with padding.This type trait tests whether the given data type empl...
Definition: IsPadded.h:76
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:102
Header file for the Or class template.
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: SMatDMatMultExpr.h:352
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
Header file for the DenseMatrix base class.
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:3085
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
decltype(auto) decllow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as lower.
Definition: DMatDeclLowExpr.h:1026
Header file for the IsLower type trait.
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:89
Generic wrapper for the null function.
Definition: Noop.h:59
Header file for the IsTriangular type trait.
Constraints on the storage order of matrix types.
Compile time check for symmetric matrices.This type trait tests whether or not the given template par...
Definition: IsSymmetric.h:85
Header file for the exception macros of the math module.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:430
Header file for the DeclDiag functor.
Constraint on the data type.
Header file for all forward declarations for expression class templates.
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: SMatDMatMultExpr.h:303
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:107
Flag for upper matrices.
Definition: SMatDMatMultExpr.h:153
Compile time check for lower unitriangular matrices.This type trait tests whether or not the given te...
Definition: IsUniLower.h:86
Header file for the conjugate shim.
MultTrait_< RT1, RT2 > ResultType
Result type for expression template evaluations.
Definition: SMatDMatMultExpr.h:244
Compile time check for resizable data types.This type trait tests whether the given data type is a re...
Definition: IsResizable.h:75
Header file for the IsSIMDCombinable type trait.
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: SMatDMatMultExpr.h:378
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
Header file for run time assertion macros.
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: SMatDMatMultExpr.h:368
LeftOperand lhs_
Left-hand side sparse matrix of the multiplication expression.
Definition: SMatDMatMultExpr.h:449
const ResultType CompositeType
Data type for composite expression templates.
Definition: SMatDMatMultExpr.h:250
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:154
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:131
Header file for the reset shim.
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:296
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:94
decltype(auto) declsym(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as symmetric.
Definition: DMatDeclSymExpr.h:1028
Compile time check for Hermitian matrices.This type trait tests whether or not the given template par...
Definition: IsHermitian.h:85
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: SMatDMatMultExpr.h:248
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: SMatDMatMultExpr.h:422
Constraints on the storage order of matrix types.
Generic wrapper for the declherm() function.
Definition: DeclHerm.h:58
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:816
Header file for the Noop functor.
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
Header file for the RemoveReference type trait.
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:224
typename T::OppositeType OppositeType_
Alias declaration for nested OppositeType type definitions.The OppositeType_ alias declaration provid...
Definition: Aliases.h:263
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:84
Generic wrapper for the declupp() function.
Definition: DeclUpp.h:58
CompositeType_< MT1 > CT1
Composite type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:133
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
const Type & ReturnType
Return type for expression template evaluations.
Definition: CompressedMatrix.h:3080
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: SMatDMatMultExpr.h:432
decltype(auto) declherm(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as Hermitian.
Definition: DMatDeclHermExpr.h:1028
Header file for the IsRowMajorMatrix type trait.
Header file for the IsComputation type trait class.
Header file for the IsBuiltin type trait.
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: SMatDMatMultExpr.h:246
Compile time logical &#39;or&#39; evaluation.The Or alias declaration performs at compile time a logical &#39;or&#39;...
Definition: Or.h:76
Flag for lower matrices.
Definition: SMatDMatMultExpr.h:152
Compile time evaluation of the size of vectors and matrices.The Size type trait evaluates the size of...
Definition: Size.h:80
IfTrue_< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side sparse matrix operand.
Definition: SMatDMatMultExpr.h:259
If_< IsExpression< MT2 >, const MT2, const MT2 &> RightOperand
Composite type of the right-hand side dense matrix expression.
Definition: SMatDMatMultExpr.h:256
Generic wrapper for the decldiag() function.
Definition: DeclDiag.h:58
If_< IsExpression< MT1 >, const MT1, const MT1 &> LeftOperand
Composite type of the left-hand side sparse matrix expression.
Definition: SMatDMatMultExpr.h:253
Header file for the DeclHerm functor.
const ElementType ReturnType
Return type for expression template evaluations.
Definition: SMatDMatMultExpr.h:249
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:628
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:423
Header file for the IsUpper type trait.
decltype(auto) conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition: DMatMapExpr.h:1321
Constraint on the data type.
Generic wrapper for the declsym() function.
Definition: DeclSym.h:58
BLAZE_ALWAYS_INLINE bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition: Matrix.h:908
Header file for the IsResizable type trait.
Header file for the Size type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the Bool class template.
Header file for the DeclSym functor.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:61
Flag for symmetric matrices.
Definition: SMatDMatMultExpr.h:150
Header file for the IsExpression type trait class.
Header file for the function trace functionality.
RightOperand rhs_
Right-hand side dense matrix of the multiplication expression.
Definition: SMatDMatMultExpr.h:450