TDMatTSMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
51 #include <blaze/math/Exception.h>
64 #include <blaze/math/shims/Reset.h>
66 #include <blaze/math/SIMD.h>
91 #include <blaze/util/Assert.h>
92 #include <blaze/util/EnableIf.h>
95 #include <blaze/util/InvalidType.h>
96 #include <blaze/util/mpl/And.h>
97 #include <blaze/util/mpl/Bool.h>
98 #include <blaze/util/mpl/If.h>
99 #include <blaze/util/mpl/Or.h>
100 #include <blaze/util/Types.h>
103 
104 
105 namespace blaze {
106 
107 //=================================================================================================
108 //
109 // CLASS TDMATTSMATMULTEXPR
110 //
111 //=================================================================================================
112 
113 //*************************************************************************************************
120 template< typename MT1 // Type of the left-hand side dense matrix
121  , typename MT2 // Type of the right-hand side sparse matrix
122  , bool SF // Symmetry flag
123  , bool HF // Hermitian flag
124  , bool LF // Lower flag
125  , bool UF > // Upper flag
126 class TDMatTSMatMultExpr
127  : public MatMatMultExpr< DenseMatrix< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, true > >
128  , private Computation
129 {
130  private:
131  //**Type definitions****************************************************************************
138  //**********************************************************************************************
139 
140  //**********************************************************************************************
142  enum : bool { evaluateLeft = IsComputation<MT1>::value || RequiresEvaluation<MT1>::value };
143  //**********************************************************************************************
144 
145  //**********************************************************************************************
147  enum : bool { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
148  //**********************************************************************************************
149 
150  //**********************************************************************************************
152  enum : bool {
153  SYM = ( SF && !( HF || LF || UF ) ),
154  HERM = ( HF && !( LF || UF ) ),
155  LOW = ( LF || ( ( SF || HF ) && UF ) ),
156  UPP = ( UF || ( ( SF || HF ) && LF ) )
157  };
158  //**********************************************************************************************
159 
160  //**********************************************************************************************
162 
166  template< typename T1, typename T2, typename T3 >
167  struct IsEvaluationRequired {
168  enum : bool { value = ( evaluateLeft || evaluateRight ) };
169  };
171  //**********************************************************************************************
172 
173  //**********************************************************************************************
175 
178  template< typename T1, typename T2, typename T3 >
179  struct UseVectorizedKernel {
180  enum : bool { value = useOptimizedKernels &&
182  T1::simdEnabled && T2::simdEnabled &&
186  , ElementType_<T3> >::value &&
189  };
191  //**********************************************************************************************
192 
193  //**********************************************************************************************
195 
199  template< typename T1, typename T2, typename T3 >
200  struct UseOptimizedKernel {
201  enum : bool { value = useOptimizedKernels &&
202  !UseVectorizedKernel<T1,T2,T3>::value &&
204  !IsResizable< ElementType_<T1> >::value &&
206  };
208  //**********************************************************************************************
209 
210  //**********************************************************************************************
212 
215  template< typename T1, typename T2, typename T3 >
216  struct UseDefaultKernel {
217  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
218  !UseOptimizedKernel<T1,T2,T3>::value };
219  };
221  //**********************************************************************************************
222 
223  //**********************************************************************************************
225 
228  using ForwardFunctor = IfTrue_< HERM
229  , DeclHerm
230  , IfTrue_< SYM
231  , DeclSym
232  , IfTrue_< LOW
233  , IfTrue_< UPP
234  , DeclDiag
235  , DeclLow >
236  , IfTrue_< UPP
237  , DeclUpp
238  , Noop > > > >;
240  //**********************************************************************************************
241 
242  public:
243  //**Type definitions****************************************************************************
246 
252  using ReturnType = const ElementType;
253  using CompositeType = const ResultType;
254 
256  using LeftOperand = If_< IsExpression<MT1>, const MT1, const MT1& >;
257 
259  using RightOperand = If_< IsExpression<MT2>, const MT2, const MT2& >;
260 
263 
266  //**********************************************************************************************
267 
268  //**Compilation flags***************************************************************************
270  enum : bool { simdEnabled = !IsDiagonal<MT1>::value &&
271  MT1::simdEnabled &&
274 
276  enum : bool { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
277  !evaluateRight && MT2::smpAssignable };
278  //**********************************************************************************************
279 
280  //**SIMD properties*****************************************************************************
282  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
283  //**********************************************************************************************
284 
285  //**Constructor*********************************************************************************
291  explicit inline TDMatTSMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
292  : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
293  , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
294  {
295  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
296  }
297  //**********************************************************************************************
298 
299  //**Access operator*****************************************************************************
306  inline ReturnType operator()( size_t i, size_t j ) const {
307  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
308  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
309 
310  if( IsDiagonal<MT1>::value ) {
311  return lhs_(i,i) * rhs_(i,j);
312  }
313  else if( IsDiagonal<MT2>::value ) {
314  return lhs_(i,j) * rhs_(j,j);
315  }
317  const size_t begin( ( IsUpper<MT1>::value )
318  ?( ( IsLower<MT2>::value )
319  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
320  , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
321  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
322  :( ( IsLower<MT2>::value )
323  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
324  :( 0UL ) ) );
325  const size_t end( ( IsLower<MT1>::value )
326  ?( ( IsUpper<MT2>::value )
327  ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
328  , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
329  :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
330  :( ( IsUpper<MT2>::value )
331  ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
332  :( lhs_.columns() ) ) );
333 
334  if( begin >= end ) return ElementType();
335 
336  const size_t n( end - begin );
337 
338  return subvector( row( lhs_, i ), begin, n ) * subvector( column( rhs_, j ), begin, n );
339  }
340  else {
341  return row( lhs_, i ) * column( rhs_, j );
342  }
343  }
344  //**********************************************************************************************
345 
346  //**At function*********************************************************************************
354  inline ReturnType at( size_t i, size_t j ) const {
355  if( i >= lhs_.rows() ) {
356  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
357  }
358  if( j >= rhs_.columns() ) {
359  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
360  }
361  return (*this)(i,j);
362  }
363  //**********************************************************************************************
364 
365  //**Rows function*******************************************************************************
370  inline size_t rows() const noexcept {
371  return lhs_.rows();
372  }
373  //**********************************************************************************************
374 
375  //**Columns function****************************************************************************
380  inline size_t columns() const noexcept {
381  return rhs_.columns();
382  }
383  //**********************************************************************************************
384 
385  //**Left operand access*************************************************************************
390  inline LeftOperand leftOperand() const noexcept {
391  return lhs_;
392  }
393  //**********************************************************************************************
394 
395  //**Right operand access************************************************************************
400  inline RightOperand rightOperand() const noexcept {
401  return rhs_;
402  }
403  //**********************************************************************************************
404 
405  //**********************************************************************************************
411  template< typename T >
412  inline bool canAlias( const T* alias ) const noexcept {
413  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
414  }
415  //**********************************************************************************************
416 
417  //**********************************************************************************************
423  template< typename T >
424  inline bool isAliased( const T* alias ) const noexcept {
425  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
426  }
427  //**********************************************************************************************
428 
429  //**********************************************************************************************
434  inline bool isAligned() const noexcept {
435  return lhs_.isAligned();
436  }
437  //**********************************************************************************************
438 
439  //**********************************************************************************************
444  inline bool canSMPAssign() const noexcept {
445  return ( rows() * columns() >= SMP_TDMATTSMATMULT_THRESHOLD ) && !IsDiagonal<MT1>::value;
446  }
447  //**********************************************************************************************
448 
449  private:
450  //**Member variables****************************************************************************
453  //**********************************************************************************************
454 
455  //**Assignment to dense matrices****************************************************************
468  template< typename MT // Type of the target dense matrix
469  , bool SO > // Storage order of the target dense matrix
470  friend inline void assign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
471  {
473 
474  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
475  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
476 
477  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
478  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
479 
480  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
481  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
482  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
483  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
484  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
485  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
486 
487  TDMatTSMatMultExpr::selectAssignKernel( ~lhs, A, B );
488  }
490  //**********************************************************************************************
491 
492  //**Default assignment to dense matrices********************************************************
506  template< typename MT3 // Type of the left-hand side target matrix
507  , typename MT4 // Type of the left-hand side matrix operand
508  , typename MT5 > // Type of the right-hand side matrix operand
510  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
511  {
513 
514  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
515 
516  reset( C );
517 
518  for( size_t ii=0UL; ii<A.rows(); ii+=block )
519  {
520  const size_t itmp( min( ii+block, A.rows() ) );
521 
522  for( size_t j=0UL; j<B.columns(); ++j )
523  {
524  ConstIterator element( B.begin(j) );
525  const ConstIterator end( B.end(j) );
526 
527  for( ; element!=end; ++element )
528  {
529  const size_t j1( element->index() );
530 
532  {
533  C(j1,j) = A(j1,j1) * element->value();
534  }
535  else
536  {
537  const size_t ibegin( ( IsLower<MT4>::value )
538  ?( ( LOW )
539  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
540  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
541  :( LOW ? max(j,ii) : ii ) );
542  const size_t iend( ( IsUpper<MT4>::value )
543  ?( ( SYM || HERM || UPP )
544  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
545  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
546  :( SYM || HERM || UPP ? min(j+1UL,itmp) : itmp ) );
547 
548  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
549  continue;
550 
551  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
552 
553  for( size_t i=ibegin; i<iend; ++i ) {
554  if( isDefault( C(i,j) ) )
555  C(i,j) = A(i,j1) * element->value();
556  else
557  C(i,j) += A(i,j1) * element->value();
558  }
559  }
560  }
561  }
562  }
563 
564  if( SYM || HERM ) {
565  for( size_t j=0UL; j<B.columns(); ++j ) {
566  for( size_t i=j+1UL; i<A.rows(); ++i ) {
567  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
568  }
569  }
570  }
571  }
573  //**********************************************************************************************
574 
575  //**Optimized assignment to dense matrices******************************************************
589  template< typename MT3 // Type of the left-hand side target matrix
590  , typename MT4 // Type of the left-hand side matrix operand
591  , typename MT5 > // Type of the right-hand side matrix operand
593  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
594  {
596 
597  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
598 
599  reset( C );
600 
601  for( size_t ii=0UL; ii<A.rows(); ii+=block )
602  {
603  const size_t itmp( min( ii+block, A.rows() ) );
604 
605  for( size_t j=0UL; j<B.columns(); ++j )
606  {
607  const ConstIterator end( B.end(j) );
608  ConstIterator element( B.begin(j) );
609 
610  const size_t nonzeros( B.nonZeros(j) );
611  const size_t kpos( nonzeros & size_t(-4) );
612  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
613 
614  for( size_t k=0UL; k<kpos; k+=4UL )
615  {
616  const size_t j1( element->index() );
617  const ET2 v1( element->value() );
618  ++element;
619  const size_t j2( element->index() );
620  const ET2 v2( element->value() );
621  ++element;
622  const size_t j3( element->index() );
623  const ET2 v3( element->value() );
624  ++element;
625  const size_t j4( element->index() );
626  const ET2 v4( element->value() );
627  ++element;
628 
629  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
630 
631  const size_t ibegin( ( IsLower<MT4>::value )
632  ?( ( LOW )
633  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
634  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
635  :( LOW ? max(j,ii) : ii ) );
636  const size_t iend( ( IsUpper<MT4>::value )
637  ?( ( SYM || HERM || UPP )
638  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
639  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) ) )
640  :( SYM || HERM || UPP ? min(j+1UL,itmp) : itmp ) );
641 
642  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
643  continue;
644 
645  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
646 
647  const size_t inum( iend - ibegin );
648  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
649  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
650 
651  for( size_t i=ibegin; i<ipos; i+=4UL ) {
652  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
653  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
654  C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
655  C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
656  }
657  for( size_t i=ipos; i<iend; ++i ) {
658  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
659  }
660  }
661 
662  for( ; element!=end; ++element )
663  {
664  const size_t j1( element->index() );
665  const ET2 v1( element->value() );
666 
667  const size_t ibegin( ( IsLower<MT4>::value )
668  ?( ( LOW )
669  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
670  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
671  :( LOW ? max(j,ii) : ii ) );
672  const size_t iend( ( IsUpper<MT4>::value )
673  ?( ( SYM || HERM || UPP )
674  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
675  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
676  :( SYM || HERM || UPP ? min(j+1UL,itmp) : itmp ) );
677 
678  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
679  continue;
680 
681  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
682 
683  const size_t inum( iend - ibegin );
684  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
685  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
686 
687  for( size_t i=ibegin; i<ipos; i+=4UL ) {
688  C(i ,j) += A(i ,j1) * v1;
689  C(i+1UL,j) += A(i+1UL,j1) * v1;
690  C(i+2UL,j) += A(i+2UL,j1) * v1;
691  C(i+3UL,j) += A(i+3UL,j1) * v1;
692  }
693  for( size_t i=ipos; i<iend; ++i ) {
694  C(i,j) += A(i,j1) * v1;
695  }
696  }
697  }
698  }
699 
700  if( SYM || HERM ) {
701  for( size_t j=0UL; j<B.columns(); ++j ) {
702  for( size_t i=j+1UL; i<A.rows(); ++i ) {
703  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
704  }
705  }
706  }
707  }
709  //**********************************************************************************************
710 
711  //**Vectorized assignment to column-major dense matrices****************************************
725  template< typename MT3 // Type of the left-hand side target matrix
726  , typename MT4 // Type of the left-hand side matrix operand
727  , typename MT5 > // Type of the right-hand side matrix operand
729  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
730  {
732 
733  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
734 
735  reset( C );
736 
737  for( size_t j=0UL; j<B.columns(); ++j )
738  {
739  const ConstIterator end( B.end(j) );
740  ConstIterator element( B.begin(j) );
741 
742  const size_t nonzeros( B.nonZeros(j) );
743  const size_t kpos( nonzeros & size_t(-4) );
744  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
745 
746  for( size_t k=0UL; k<kpos; k+=4UL )
747  {
748  const size_t j1( element->index() );
749  const ET2 v1( element->value() );
750  ++element;
751  const size_t j2( element->index() );
752  const ET2 v2( element->value() );
753  ++element;
754  const size_t j3( element->index() );
755  const ET2 v3( element->value() );
756  ++element;
757  const size_t j4( element->index() );
758  const ET2 v4( element->value() );
759  ++element;
760 
761  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
762 
763  const SIMDType xmm1( set( v1 ) );
764  const SIMDType xmm2( set( v2 ) );
765  const SIMDType xmm3( set( v3 ) );
766  const SIMDType xmm4( set( v4 ) );
767 
768  const size_t ibegin( ( IsLower<MT4>::value )
770  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
771  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
772  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
773  const size_t iend( ( IsUpper<MT4>::value )
775  ?( SYM || HERM || UPP ? max(j+1UL,j4) : j4 )
776  :( SYM || HERM || UPP ? max(j,j4)+1UL : j4+1UL ) )
777  :( SYM || HERM || UPP ? j+1UL : A.rows() ) );
778  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
779 
780  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
781  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
782 
783  size_t i( ibegin );
784 
785  for( ; i<ipos; i+=SIMDSIZE ) {
786  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
787  }
788  for( ; remainder && i<iend; ++i ) {
789  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
790  }
791  }
792 
793  for( ; element!=end; ++element )
794  {
795  const size_t j1( element->index() );
796  const ET2 v1( element->value() );
797 
798  const SIMDType xmm1( set( v1 ) );
799 
800  const size_t ibegin( ( IsLower<MT4>::value )
802  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
803  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
804  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
805  const size_t iend( ( IsUpper<MT4>::value )
807  ?( SYM || HERM || UPP ? max(j+1UL,j1) : j1 )
808  :( SYM || HERM || UPP ? max(j,j1)+1UL : j1+1UL ) )
809  :( SYM || HERM || UPP ? j+1UL : A.rows() ) );
810  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
811 
812  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
813  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
814 
815  size_t i( ibegin );
816 
817  for( ; i<ipos; i+=SIMDSIZE ) {
818  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 );
819  }
820  for( ; remainder && i<iend; ++i ) {
821  C(i,j) += A(i,j1) * v1;
822  }
823  }
824  }
825 
826  if( SYM || HERM ) {
827  for( size_t j=0UL; j<B.columns(); ++j ) {
828  for( size_t i=j+1UL; i<A.rows(); ++i ) {
829  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
830  }
831  }
832  }
833  }
835  //**********************************************************************************************
836 
837  //**Assignment to sparse matrices***************************************************************
850  template< typename MT // Type of the target sparse matrix
851  , bool SO > // Storage order of the target sparse matrix
852  friend inline void assign( SparseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
853  {
855 
857 
864 
865  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
866  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
867 
868  const ForwardFunctor fwd;
869 
870  const TmpType tmp( serial( rhs ) );
871  assign( ~lhs, fwd( tmp ) );
872  }
874  //**********************************************************************************************
875 
876  //**Addition assignment to dense matrices*******************************************************
889  template< typename MT // Type of the target dense matrix
890  , bool SO > // Storage order of the target dense matrix
891  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
892  {
894 
895  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
896  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
897 
898  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
899  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
900 
901  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
902  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
903  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
904  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
905  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
906  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
907 
908  TDMatTSMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
909  }
911  //**********************************************************************************************
912 
913  //**Default addition assignment to dense matrices***********************************************
927  template< typename MT3 // Type of the left-hand side target matrix
928  , typename MT4 // Type of the left-hand side matrix operand
929  , typename MT5 > // Type of the right-hand side matrix operand
931  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
932  {
934 
935  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
936 
937  for( size_t ii=0UL; ii<A.rows(); ii+=block )
938  {
939  const size_t itmp( min( ii+block, A.rows() ) );
940 
941  for( size_t j=0UL; j<B.columns(); ++j )
942  {
943  ConstIterator element( B.begin(j) );
944  const ConstIterator end( B.end(j) );
945 
946  for( ; element!=end; ++element )
947  {
948  const size_t j1( element->index() );
949 
951  {
952  C(j1,j) += A(j1,j1) * element->value();
953  }
954  else
955  {
956  const size_t ibegin( ( IsLower<MT4>::value )
957  ?( ( LOW )
958  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
959  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
960  :( LOW ? max(j,ii) : ii ) );
961  const size_t iend( ( IsUpper<MT4>::value )
962  ?( ( UPP )
963  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
964  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
965  :( UPP ? min(j+1UL,itmp) : itmp ) );
966 
967  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
968  continue;
969 
970  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
971 
972  const size_t inum( iend - ibegin );
973  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
974  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
975 
976  for( size_t i=ibegin; i<ipos; i+=4UL ) {
977  C(i ,j) += A(i ,j1) * element->value();
978  C(i+1UL,j) += A(i+1UL,j1) * element->value();
979  C(i+2UL,j) += A(i+2UL,j1) * element->value();
980  C(i+3UL,j) += A(i+3UL,j1) * element->value();
981  }
982  for( size_t i=ipos; i<iend; ++i ) {
983  C(i,j) += A(i,j1) * element->value();
984  }
985  }
986  }
987  }
988  }
989  }
991  //**********************************************************************************************
992 
993  //**Optimized addition assignment to dense matrices*********************************************
1007  template< typename MT3 // Type of the left-hand side target matrix
1008  , typename MT4 // Type of the left-hand side matrix operand
1009  , typename MT5 > // Type of the right-hand side matrix operand
1011  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1012  {
1014 
1015  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
1016 
1017  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1018  {
1019  const size_t itmp( min( ii+block, A.rows() ) );
1020 
1021  for( size_t j=0UL; j<B.columns(); ++j )
1022  {
1023  const ConstIterator end( B.end(j) );
1024  ConstIterator element( B.begin(j) );
1025 
1026  const size_t nonzeros( B.nonZeros(j) );
1027  const size_t kpos( nonzeros & size_t(-4) );
1028  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1029 
1030  for( size_t k=0UL; k<kpos; k+=4UL )
1031  {
1032  const size_t j1( element->index() );
1033  const ET2 v1( element->value() );
1034  ++element;
1035  const size_t j2( element->index() );
1036  const ET2 v2( element->value() );
1037  ++element;
1038  const size_t j3( element->index() );
1039  const ET2 v3( element->value() );
1040  ++element;
1041  const size_t j4( element->index() );
1042  const ET2 v4( element->value() );
1043  ++element;
1044 
1045  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1046 
1047  const size_t ibegin( ( IsLower<MT4>::value )
1048  ?( ( LOW )
1049  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1050  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1051  :( LOW ? max(j,ii) : ii ) );
1052  const size_t iend( ( IsUpper<MT4>::value )
1053  ?( ( UPP )
1054  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
1055  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) ) )
1056  :( UPP ? min(j+1UL,itmp) : itmp ) );
1057 
1058  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1059  continue;
1060 
1061  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1062 
1063  const size_t inum( iend - ibegin );
1064  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1065  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1066 
1067  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1068  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1069  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1070  C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1071  C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1072  }
1073  for( size_t i=ipos; i<iend; ++i ) {
1074  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1075  }
1076  }
1077 
1078  for( ; element!=end; ++element )
1079  {
1080  const size_t j1( element->index() );
1081  const ET2 v1( element->value() );
1082 
1083  const size_t ibegin( ( IsLower<MT4>::value )
1084  ?( ( LOW )
1085  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1086  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1087  :( LOW ? max(j,ii) : ii ) );
1088  const size_t iend( ( IsUpper<MT4>::value )
1089  ?( ( UPP )
1090  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1091  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
1092  :( UPP ? min(j+1UL,itmp) : itmp ) );
1093 
1094  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1095  continue;
1096 
1097  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1098 
1099  const size_t inum( iend - ibegin );
1100  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1101  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1102 
1103  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1104  C(i ,j) += A(i ,j1) * v1;
1105  C(i+1UL,j) += A(i+1UL,j1) * v1;
1106  C(i+2UL,j) += A(i+2UL,j1) * v1;
1107  C(i+3UL,j) += A(i+3UL,j1) * v1;
1108  }
1109  for( size_t i=ipos; i<iend; ++i ) {
1110  C(i,j) += A(i,j1) * v1;
1111  }
1112  }
1113  }
1114  }
1115  }
1117  //**********************************************************************************************
1118 
1119  //**Vectorized addition assignment to column-major dense matrices*******************************
1133  template< typename MT3 // Type of the left-hand side target matrix
1134  , typename MT4 // Type of the left-hand side matrix operand
1135  , typename MT5 > // Type of the right-hand side matrix operand
1137  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1138  {
1140 
1141  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
1142 
1143  for( size_t j=0UL; j<B.columns(); ++j )
1144  {
1145  const ConstIterator end( B.end(j) );
1146  ConstIterator element( B.begin(j) );
1147 
1148  const size_t nonzeros( B.nonZeros(j) );
1149  const size_t kpos( nonzeros & size_t(-4) );
1150  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1151 
1152  for( size_t k=0UL; k<kpos; k+=4UL )
1153  {
1154  const size_t j1( element->index() );
1155  const ET2 v1( element->value() );
1156  ++element;
1157  const size_t j2( element->index() );
1158  const ET2 v2( element->value() );
1159  ++element;
1160  const size_t j3( element->index() );
1161  const ET2 v3( element->value() );
1162  ++element;
1163  const size_t j4( element->index() );
1164  const ET2 v4( element->value() );
1165  ++element;
1166 
1167  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1168 
1169  const SIMDType xmm1( set( v1 ) );
1170  const SIMDType xmm2( set( v2 ) );
1171  const SIMDType xmm3( set( v3 ) );
1172  const SIMDType xmm4( set( v4 ) );
1173 
1174  const size_t ibegin( ( IsLower<MT4>::value )
1176  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
1177  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
1178  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
1179  const size_t iend( ( IsUpper<MT4>::value )
1181  ?( UPP ? max(j+1UL,j4) : j4 )
1182  :( UPP ? max(j,j4)+1UL : j4+1UL ) )
1183  :( UPP ? j+1UL : A.rows() ) );
1184  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1185 
1186  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1187  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1188 
1189  size_t i( ibegin );
1190 
1191  for( ; i<ipos; i+=SIMDSIZE ) {
1192  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
1193  }
1194  for( ; remainder && i<iend; ++i ) {
1195  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1196  }
1197  }
1198 
1199  for( ; element!=end; ++element )
1200  {
1201  const size_t j1( element->index() );
1202  const ET2 v1( element->value() );
1203 
1204  const SIMDType xmm1( set( v1 ) );
1205 
1206  const size_t ibegin( ( IsLower<MT4>::value )
1208  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
1209  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
1210  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
1211  const size_t iend( ( IsUpper<MT4>::value )
1213  ?( UPP ? max(j+1UL,j1) : j1 )
1214  :( UPP ? max(j,j1)+1UL : j1+1UL ) )
1215  :( UPP ? j+1UL : A.rows() ) );
1216  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1217 
1218  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1219  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1220 
1221  size_t i( ibegin );
1222 
1223  for( ; i<ipos; i+=SIMDSIZE ) {
1224  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 );
1225  }
1226  for( ; remainder && i<iend; ++i ) {
1227  C(i,j) += A(i,j1) * v1;
1228  }
1229  }
1230  }
1231  }
1233  //**********************************************************************************************
1234 
1235  //**Addition assignment to sparse matrices******************************************************
1236  // No special implementation for the addition assignment to sparse matrices.
1237  //**********************************************************************************************
1238 
1239  //**Subtraction assignment to dense matrices****************************************************
1252  template< typename MT // Type of the target dense matrix
1253  , bool SO > // Storage order of the target dense matrix
1254  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1255  {
1257 
1258  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1259  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1260 
1261  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1262  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1263 
1264  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1265  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1266  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1267  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1268  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1269  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1270 
1271  TDMatTSMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1272  }
1274  //**********************************************************************************************
1275 
1276  //**Default subtraction assignment to dense matrices********************************************
1290  template< typename MT3 // Type of the left-hand side target matrix
1291  , typename MT4 // Type of the left-hand side matrix operand
1292  , typename MT5 > // Type of the right-hand side matrix operand
1294  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1295  {
1297 
1298  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
1299 
1300  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1301  {
1302  const size_t itmp( min( ii+block, A.rows() ) );
1303 
1304  for( size_t j=0UL; j<B.columns(); ++j )
1305  {
1306  ConstIterator element( B.begin(j) );
1307  const ConstIterator end( B.end(j) );
1308 
1309  for( ; element!=end; ++element )
1310  {
1311  const size_t j1( element->index() );
1312 
1314  {
1315  C(j1,j) -= A(j1,j1) * element->value();
1316  }
1317  else
1318  {
1319  const size_t ibegin( ( IsLower<MT4>::value )
1320  ?( ( LOW )
1321  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1322  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1323  :( LOW ? max(j,ii) : ii ) );
1324  const size_t iend( ( IsUpper<MT4>::value )
1325  ?( ( UPP )
1326  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1327  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
1328  :( UPP ? min(j+1UL,itmp) : itmp ) );
1329 
1330  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1331  continue;
1332 
1333  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1334 
1335  const size_t inum( iend - ibegin );
1336  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1337  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1338 
1339  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1340  C(i ,j) -= A(i ,j1) * element->value();
1341  C(i+1UL,j) -= A(i+1UL,j1) * element->value();
1342  C(i+2UL,j) -= A(i+2UL,j1) * element->value();
1343  C(i+3UL,j) -= A(i+3UL,j1) * element->value();
1344  }
1345  for( size_t i=ipos; i<iend; ++i ) {
1346  C(i,j) -= A(i,j1) * element->value();
1347  }
1348  }
1349  }
1350  }
1351  }
1352  }
1354  //**********************************************************************************************
1355 
1356  //**Optimized subtraction assignment to dense matrices******************************************
1370  template< typename MT3 // Type of the left-hand side target matrix
1371  , typename MT4 // Type of the left-hand side matrix operand
1372  , typename MT5 > // Type of the right-hand side matrix operand
1374  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1375  {
1377 
1378  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
1379 
1380  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1381  {
1382  const size_t itmp( min( ii+block, A.rows() ) );
1383 
1384  for( size_t j=0UL; j<B.columns(); ++j )
1385  {
1386  const ConstIterator end( B.end(j) );
1387  ConstIterator element( B.begin(j) );
1388 
1389  const size_t nonzeros( B.nonZeros(j) );
1390  const size_t kpos( nonzeros & size_t(-4) );
1391  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1392 
1393  for( size_t k=0UL; k<kpos; k+=4UL )
1394  {
1395  const size_t j1( element->index() );
1396  const ET2 v1( element->value() );
1397  ++element;
1398  const size_t j2( element->index() );
1399  const ET2 v2( element->value() );
1400  ++element;
1401  const size_t j3( element->index() );
1402  const ET2 v3( element->value() );
1403  ++element;
1404  const size_t j4( element->index() );
1405  const ET2 v4( element->value() );
1406  ++element;
1407 
1408  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1409 
1410  const size_t ibegin( ( IsLower<MT4>::value )
1411  ?( ( LOW )
1412  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1413  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1414  :( LOW ? max(j,ii) : ii ) );
1415  const size_t iend( ( IsUpper<MT4>::value )
1416  ?( ( UPP )
1417  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
1418  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) ) )
1419  :( UPP ? min(j+1UL,itmp) : itmp ) );
1420 
1421  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1422  continue;
1423 
1424  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1425 
1426  const size_t inum( iend - ibegin );
1427  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1428  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1429 
1430  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1431  C(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1432  C(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1433  C(i+2UL,j) -= A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1434  C(i+3UL,j) -= A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1435  }
1436  for( size_t i=ipos; i<iend; ++i ) {
1437  C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1438  }
1439  }
1440 
1441  for( ; element!=end; ++element )
1442  {
1443  const size_t j1( element->index() );
1444  const ET2 v1( element->value() );
1445 
1446  const size_t ibegin( ( IsLower<MT4>::value )
1447  ?( ( LOW )
1448  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1449  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1450  :( LOW ? max(j,ii) : ii ) );
1451  const size_t iend( ( IsUpper<MT4>::value )
1452  ?( ( UPP )
1453  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1454  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
1455  :( UPP ? min(j+1UL,itmp) : itmp ) );
1456 
1457  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1458  continue;
1459 
1460  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1461 
1462  const size_t inum( iend - ibegin );
1463  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1464  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1465 
1466  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1467  C(i ,j) -= A(i ,j1) * v1;
1468  C(i+1UL,j) -= A(i+1UL,j1) * v1;
1469  C(i+2UL,j) -= A(i+2UL,j1) * v1;
1470  C(i+3UL,j) -= A(i+3UL,j1) * v1;
1471  }
1472  for( size_t i=ipos; i<iend; ++i ) {
1473  C(i,j) -= A(i,j1) * v1;
1474  }
1475  }
1476  }
1477  }
1478  }
1480  //**********************************************************************************************
1481 
1482  //**Vectorized subtraction assignment to column-major dense matrices****************************
1496  template< typename MT3 // Type of the left-hand side target matrix
1497  , typename MT4 // Type of the left-hand side matrix operand
1498  , typename MT5 > // Type of the right-hand side matrix operand
1500  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1501  {
1503 
1504  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
1505 
1506  for( size_t j=0UL; j<B.columns(); ++j )
1507  {
1508  const ConstIterator end( B.end(j) );
1509  ConstIterator element( B.begin(j) );
1510 
1511  const size_t nonzeros( B.nonZeros(j) );
1512  const size_t kpos( nonzeros & size_t(-4) );
1513  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1514 
1515  for( size_t k=0UL; k<kpos; k+=4UL )
1516  {
1517  const size_t j1( element->index() );
1518  const ET2 v1( element->value() );
1519  ++element;
1520  const size_t j2( element->index() );
1521  const ET2 v2( element->value() );
1522  ++element;
1523  const size_t j3( element->index() );
1524  const ET2 v3( element->value() );
1525  ++element;
1526  const size_t j4( element->index() );
1527  const ET2 v4( element->value() );
1528  ++element;
1529 
1530  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1531 
1532  const SIMDType xmm1( set( v1 ) );
1533  const SIMDType xmm2( set( v2 ) );
1534  const SIMDType xmm3( set( v3 ) );
1535  const SIMDType xmm4( set( v4 ) );
1536 
1537  const size_t ibegin( ( IsLower<MT4>::value )
1539  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
1540  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
1541  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
1542  const size_t iend( ( IsUpper<MT4>::value )
1544  ?( UPP ? max(j+1UL,j4) : j4 )
1545  :( UPP ? max(j,j4)+1UL : j4+1UL ) )
1546  :( UPP ? j+1UL : A.rows() ) );
1547  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1548 
1549  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1550  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1551 
1552  size_t i( ibegin );
1553 
1554  for( ; i<ipos; i+=SIMDSIZE ) {
1555  C.store( i, j, C.load(i,j) - A.load(i,j1) * xmm1 - A.load(i,j2) * xmm2 - A.load(i,j3) * xmm3 - A.load(i,j4) * xmm4 );
1556  }
1557  for( ; remainder && i<iend; ++i ) {
1558  C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1559  }
1560  }
1561 
1562  for( ; element!=end; ++element )
1563  {
1564  const size_t j1( element->index() );
1565  const ET2 v1( element->value() );
1566 
1567  const SIMDType xmm1( set( v1 ) );
1568 
1569  const size_t ibegin( ( IsLower<MT4>::value )
1571  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
1572  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
1573  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
1574  const size_t iend( ( IsUpper<MT4>::value )
1576  ?( UPP ? max(j+1UL,j1) : j1 )
1577  :( UPP ? max(j,j1)+1UL : j1+1UL ) )
1578  :( UPP ? j+1UL : A.rows() ) );
1579  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1580 
1581  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1582  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1583 
1584  size_t i( ibegin );
1585 
1586  for( ; i<ipos; i+=SIMDSIZE ) {
1587  C.store( i, j, C.load(i,j) - A.load(i,j1) * xmm1 );
1588  }
1589  for( ; remainder && i<iend; ++i ) {
1590  C(i,j) -= A(i,j1) * v1;
1591  }
1592  }
1593  }
1594  }
1596  //**********************************************************************************************
1597 
1598  //**Subtraction assignment to sparse matrices***************************************************
1599  // No special implementation for the subtraction assignment to sparse matrices.
1600  //**********************************************************************************************
1601 
1602  //**Schur product assignment to dense matrices**************************************************
1615  template< typename MT // Type of the target dense matrix
1616  , bool SO > // Storage order of the target dense matrix
1617  friend inline void schurAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1618  {
1620 
1624 
1625  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1626  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1627 
1628  const ResultType tmp( serial( rhs ) );
1629  schurAssign( ~lhs, tmp );
1630  }
1632  //**********************************************************************************************
1633 
1634  //**Schur product assignment to sparse matrices*************************************************
1635  // No special implementation for the Schur product assignment to sparse matrices.
1636  //**********************************************************************************************
1637 
1638  //**Multiplication assignment to dense matrices*************************************************
1639  // No special implementation for the multiplication assignment to dense matrices.
1640  //**********************************************************************************************
1641 
1642  //**Multiplication assignment to sparse matrices************************************************
1643  // No special implementation for the multiplication assignment to sparse matrices.
1644  //**********************************************************************************************
1645 
1646  //**SMP assignment to dense matrices************************************************************
1661  template< typename MT // Type of the target dense matrix
1662  , bool SO > // Storage order of the target dense matrix
1664  smpAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1665  {
1667 
1668  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1669  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1670 
1671  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1672  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1673 
1674  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1675  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1676  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1677  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1678  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1679  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1680 
1681  smpAssign( ~lhs, A * B );
1682  }
1684  //**********************************************************************************************
1685 
1686  //**SMP assignment to sparse matrices***********************************************************
1701  template< typename MT // Type of the target sparse matrix
1702  , bool SO > // Storage order of the target sparse matrix
1705  {
1707 
1709 
1716 
1717  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1718  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1719 
1720  const ForwardFunctor fwd;
1721 
1722  const TmpType tmp( rhs );
1723  smpAssign( ~lhs, fwd( tmp ) );
1724  }
1726  //**********************************************************************************************
1727 
1728  //**SMP addition assignment to dense matrices***************************************************
1743  template< typename MT // Type of the target dense matrix
1744  , bool SO > // Storage order of the target dense matrix
1747  {
1749 
1750  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1751  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1752 
1753  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1754  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1755 
1756  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1757  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1758  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1759  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1760  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1761  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1762 
1763  smpAddAssign( ~lhs, A * B );
1764  }
1766  //**********************************************************************************************
1767 
1768  //**SMP addition assignment to sparse matrices**************************************************
1769  // No special implementation for the SMP addition assignment to sparse matrices.
1770  //**********************************************************************************************
1771 
1772  //**SMP subtraction assignment to dense matrices************************************************
1787  template< typename MT // Type of the target dense matrix
1788  , bool SO > // Storage order of the target dense matrix
1791  {
1793 
1794  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1795  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1796 
1797  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1798  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1799 
1800  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1801  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1802  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1803  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1804  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1805  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1806 
1807  smpSubAssign( ~lhs, A * B );
1808  }
1810  //**********************************************************************************************
1811 
1812  //**SMP subtraction assignment to sparse matrices***********************************************
1813  // No special implementation for the SMP subtraction assignment to sparse matrices.
1814  //**********************************************************************************************
1815 
1816  //**SMP Schur product assignment to dense matrices**********************************************
1829  template< typename MT // Type of the target dense matrix
1830  , bool SO > // Storage order of the target dense matrix
1831  friend inline void smpSchurAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1832  {
1834 
1838 
1839  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1840  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1841 
1842  const ResultType tmp( rhs );
1843  smpSchurAssign( ~lhs, tmp );
1844  }
1846  //**********************************************************************************************
1847 
1848  //**SMP Schur product assignment to sparse matrices*********************************************
1849  // No special implementation for the SMP Schur product assignment to sparse matrices.
1850  //**********************************************************************************************
1851 
1852  //**SMP multiplication assignment to dense matrices*********************************************
1853  // No special implementation for the SMP multiplication assignment to dense matrices.
1854  //**********************************************************************************************
1855 
1856  //**SMP multiplication assignment to sparse matrices********************************************
1857  // No special implementation for the SMP multiplication assignment to sparse matrices.
1858  //**********************************************************************************************
1859 
1860  //**Compile time checks*************************************************************************
1868  //**********************************************************************************************
1869 };
1870 //*************************************************************************************************
1871 
1872 
1873 
1874 
1875 //=================================================================================================
1876 //
1877 // GLOBAL BINARY ARITHMETIC OPERATORS
1878 //
1879 //=================================================================================================
1880 
1881 //*************************************************************************************************
1910 template< typename MT1 // Type of the left-hand side dense matrix
1911  , typename MT2 > // Type of the right-hand side sparse matrix
1912 inline decltype(auto)
1913  operator*( const DenseMatrix<MT1,true>& lhs, const SparseMatrix<MT2,true>& rhs )
1914 {
1916 
1917  if( (~lhs).columns() != (~rhs).rows() ) {
1918  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1919  }
1920 
1922  return ReturnType( ~lhs, ~rhs );
1923 }
1924 //*************************************************************************************************
1925 
1926 
1927 
1928 
1929 //=================================================================================================
1930 //
1931 // GLOBAL FUNCTIONS
1932 //
1933 //=================================================================================================
1934 
1935 //*************************************************************************************************
1959 template< typename MT1 // Type of the left-hand side dense matrix
1960  , typename MT2 // Type of the right-hand side dense matrix
1961  , bool SF // Symmetry flag
1962  , bool HF // Hermitian flag
1963  , bool LF // Lower flag
1964  , bool UF > // Upper flag
1965 inline decltype(auto) declsym( const TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
1966 {
1968 
1969  if( !isSquare( dm ) ) {
1970  BLAZE_THROW_INVALID_ARGUMENT( "Invalid symmetric matrix specification" );
1971  }
1972 
1974  return ReturnType( dm.leftOperand(), dm.rightOperand() );
1975 }
1977 //*************************************************************************************************
1978 
1979 
1980 //*************************************************************************************************
2004 template< typename MT1 // Type of the left-hand side dense matrix
2005  , typename MT2 // Type of the right-hand side dense matrix
2006  , bool SF // Symmetry flag
2007  , bool HF // Hermitian flag
2008  , bool LF // Lower flag
2009  , bool UF > // Upper flag
2010 inline decltype(auto) declherm( const TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2011 {
2013 
2014  if( !isSquare( dm ) ) {
2015  BLAZE_THROW_INVALID_ARGUMENT( "Invalid Hermitian matrix specification" );
2016  }
2017 
2019  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2020 }
2022 //*************************************************************************************************
2023 
2024 
2025 //*************************************************************************************************
2049 template< typename MT1 // Type of the left-hand side dense matrix
2050  , typename MT2 // Type of the right-hand side dense matrix
2051  , bool SF // Symmetry flag
2052  , bool HF // Hermitian flag
2053  , bool LF // Lower flag
2054  , bool UF > // Upper flag
2055 inline decltype(auto) decllow( const TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2056 {
2058 
2059  if( !isSquare( dm ) ) {
2060  BLAZE_THROW_INVALID_ARGUMENT( "Invalid lower matrix specification" );
2061  }
2062 
2064  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2065 }
2067 //*************************************************************************************************
2068 
2069 
2070 //*************************************************************************************************
2094 template< typename MT1 // Type of the left-hand side dense matrix
2095  , typename MT2 // Type of the right-hand side dense matrix
2096  , bool SF // Symmetry flag
2097  , bool HF // Hermitian flag
2098  , bool LF // Lower flag
2099  , bool UF > // Upper flag
2100 inline decltype(auto) declupp( const TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2101 {
2103 
2104  if( !isSquare( dm ) ) {
2105  BLAZE_THROW_INVALID_ARGUMENT( "Invalid upper matrix specification" );
2106  }
2107 
2109  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2110 }
2112 //*************************************************************************************************
2113 
2114 
2115 //*************************************************************************************************
2139 template< typename MT1 // Type of the left-hand side dense matrix
2140  , typename MT2 // Type of the right-hand side dense matrix
2141  , bool SF // Symmetry flag
2142  , bool HF // Hermitian flag
2143  , bool LF // Lower flag
2144  , bool UF > // Upper flag
2145 inline decltype(auto) decldiag( const TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2146 {
2148 
2149  if( !isSquare( dm ) ) {
2150  BLAZE_THROW_INVALID_ARGUMENT( "Invalid diagonal matrix specification" );
2151  }
2152 
2154  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2155 }
2157 //*************************************************************************************************
2158 
2159 
2160 
2161 
2162 //=================================================================================================
2163 //
2164 // ROWS SPECIALIZATIONS
2165 //
2166 //=================================================================================================
2167 
2168 //*************************************************************************************************
2170 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2171 struct Rows< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2172  : public Rows<MT1>
2173 {};
2175 //*************************************************************************************************
2176 
2177 
2178 
2179 
2180 //=================================================================================================
2181 //
2182 // COLUMNS SPECIALIZATIONS
2183 //
2184 //=================================================================================================
2185 
2186 //*************************************************************************************************
2188 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2189 struct Columns< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2190  : public Columns<MT2>
2191 {};
2193 //*************************************************************************************************
2194 
2195 
2196 
2197 
2198 //=================================================================================================
2199 //
2200 // ISALIGNED SPECIALIZATIONS
2201 //
2202 //=================================================================================================
2203 
2204 //*************************************************************************************************
2206 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2207 struct IsAligned< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2208  : public BoolConstant< IsAligned<MT1>::value >
2209 {};
2211 //*************************************************************************************************
2212 
2213 
2214 
2215 
2216 //=================================================================================================
2217 //
2218 // ISSYMMETRIC SPECIALIZATIONS
2219 //
2220 //=================================================================================================
2221 
2222 //*************************************************************************************************
2224 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2225 struct IsSymmetric< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2226  : public BoolConstant< Or< Bool<SF>
2227  , And< Bool<HF>
2228  , IsBuiltin< ElementType_< TDMatTSMatMultExpr<MT1,MT2,false,true,false,false> > > >
2229  , And< Bool<LF>, Bool<UF> > >::value >
2230 {};
2232 //*************************************************************************************************
2233 
2234 
2235 
2236 
2237 //=================================================================================================
2238 //
2239 // ISHERMITIAN SPECIALIZATIONS
2240 //
2241 //=================================================================================================
2242 
2243 //*************************************************************************************************
2245 template< typename MT1, typename MT2, bool SF, bool LF, bool UF >
2246 struct IsHermitian< TDMatTSMatMultExpr<MT1,MT2,SF,true,LF,UF> >
2247  : public TrueType
2248 {};
2250 //*************************************************************************************************
2251 
2252 
2253 
2254 
2255 //=================================================================================================
2256 //
2257 // ISLOWER SPECIALIZATIONS
2258 //
2259 //=================================================================================================
2260 
2261 //*************************************************************************************************
2263 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2264 struct IsLower< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2265  : public BoolConstant< Or< Bool<LF>
2266  , And< IsLower<MT1>, IsLower<MT2> >
2267  , And< Or< Bool<SF>, Bool<HF> >
2268  , IsUpper<MT1>, IsUpper<MT2> > >::value >
2269 {};
2271 //*************************************************************************************************
2272 
2273 
2274 
2275 
2276 //=================================================================================================
2277 //
2278 // ISUNILOWER SPECIALIZATIONS
2279 //
2280 //=================================================================================================
2281 
2282 //*************************************************************************************************
2284 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2285 struct IsUniLower< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2286  : public BoolConstant< Or< And< IsUniLower<MT1>, IsUniLower<MT2> >
2287  , And< Or< Bool<SF>, Bool<HF> >
2288  , IsUniUpper<MT1>, IsUniUpper<MT2> > >::value >
2289 {};
2291 //*************************************************************************************************
2292 
2293 
2294 
2295 
2296 //=================================================================================================
2297 //
2298 // ISSTRICTLYLOWER SPECIALIZATIONS
2299 //
2300 //=================================================================================================
2301 
2302 //*************************************************************************************************
2304 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2305 struct IsStrictlyLower< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2306  : public BoolConstant< Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2307  , And< IsStrictlyLower<MT2>, IsLower<MT1> >
2308  , And< Or< Bool<SF>, Bool<HF> >
2309  , Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2310  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > > > >::value >
2311 {};
2313 //*************************************************************************************************
2314 
2315 
2316 
2317 
2318 //=================================================================================================
2319 //
2320 // ISUPPER SPECIALIZATIONS
2321 //
2322 //=================================================================================================
2323 
2324 //*************************************************************************************************
2326 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2327 struct IsUpper< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2328  : public BoolConstant< Or< Bool<UF>
2329  , And< IsUpper<MT1>, IsUpper<MT2> >
2330  , And< Or< Bool<SF>, Bool<HF> >
2331  , IsLower<MT1>, IsLower<MT2> > >::value >
2332 {};
2334 //*************************************************************************************************
2335 
2336 
2337 
2338 
2339 //=================================================================================================
2340 //
2341 // ISUNIUPPER SPECIALIZATIONS
2342 //
2343 //=================================================================================================
2344 
2345 //*************************************************************************************************
2347 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2348 struct IsUniUpper< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2349  : public BoolConstant< Or< And< IsUniUpper<MT1>, IsUniUpper<MT2> >
2350  , And< Or< Bool<SF>, Bool<HF> >
2351  , IsUniLower<MT1>, IsUniLower<MT2> > >::value >
2352 {};
2354 //*************************************************************************************************
2355 
2356 
2357 
2358 
2359 //=================================================================================================
2360 //
2361 // ISSTRICTLYUPPER SPECIALIZATIONS
2362 //
2363 //=================================================================================================
2364 
2365 //*************************************************************************************************
2367 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2368 struct IsStrictlyUpper< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2369  : public BoolConstant< Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2370  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> >
2371  , And< Or< Bool<SF>, Bool<HF> >
2372  , Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2373  , And< IsStrictlyLower<MT2>, IsLower<MT1> > > > >::value >
2374 {};
2376 //*************************************************************************************************
2377 
2378 } // namespace blaze
2379 
2380 #endif
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Header file for auxiliary alias declarations.
Headerfile for the generic min algorithm.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:72
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: TDMatTSMatMultExpr.h:452
RightOperand rightOperand() const noexcept
Returns the right-hand side transpose sparse matrix operand.
Definition: TDMatTSMatMultExpr.h:400
decltype(auto) decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as diagonal.
Definition: DMatDeclDiagExpr.h:996
Header file for the Rows type trait.
Header file for the IsUniUpper type trait.
EnableIf_< IsDenseMatrix< MT1 > > smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:196
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:87
Header file for basic type definitions.
Subvector< VT, AF > subvector(Vector< VT, TF > &vector, size_t index, size_t size)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:322
Expression object for transpose dense matrix-transpose sparse matrix multiplications.The TDMatTSMatMultExpr class represents the compile time expression for multiplications between a column-major dense matrix and a column-major sparse matrix.
Definition: Forward.h:155
LeftOperand leftOperand() const noexcept
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatTSMatMultExpr.h:390
EnableIf_< IsDenseMatrix< MT1 > > smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:164
IfTrue_< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatTSMatMultExpr.h:262
Header file for the serial shim.
Header file for the IsDiagonal type trait.
Generic wrapper for a compile time constant integral value.The IntegralConstant class template repres...
Definition: IntegralConstant.h:71
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
CompositeType_< MT2 > CT2
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:137
Header file for the DeclUpp functor.
ElementType_< RT2 > ET2
Element type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:135
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:198
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:172
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:316
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TDMatTSMatMultExpr.h:424
Header file for the IsColumnMajorMatrix type trait.
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:560
Header file for the And class template.
const ElementType_< MT > min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1762
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:88
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:171
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TDMatTSMatMultExpr.h:412
decltype(auto) declupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as upper.
Definition: DMatDeclUppExpr.h:1027
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:250
Column< MT > column(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific column of the given matrix.
Definition: Column.h:124
Header file for the Computation base class.
If_< IsExpression< MT2 >, const MT2, const MT2 &> RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:259
Header file for the MatMatMultExpr base class.
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:88
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Compile time check for data types.This type trait tests whether or not the given types can be combine...
Definition: IsSIMDCombinable.h:120
Header file for the IsUniLower type trait.
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:343
const ElementType_< MT > max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1809
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: TDMatTSMatMultExpr.h:251
EnableIf_< IsDenseMatrix< MT1 > > smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:133
MultTrait_< RT1, RT2 > ResultType
Result type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:247
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:78
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:129
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
ResultType_< MT2 > RT2
Result type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:133
Row< MT > row(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific row of the given matrix.
Definition: Row.h:124
Compile time check for the alignment of data types.This type trait tests whether the given data type ...
Definition: IsAligned.h:87
Constraint on the data type.
Constraint on the data type.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:72
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
Compile time check for upper unitriangular matrices.This type trait tests whether or not the given te...
Definition: IsUniUpper.h:86
Headerfile for the generic max algorithm.
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: TDMatTSMatMultExpr.h:380
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: TDMatTSMatMultExpr.h:370
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
Header file for the DeclLow functor.
Header file for the If class template.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
TDMatTSMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the TDMatTSMatMultExpr class.
Definition: TDMatTSMatMultExpr.h:291
Generic wrapper for the decllow() function.
Definition: DeclLow.h:58
Compile time check for data types with padding.This type trait tests whether the given data type empl...
Definition: IsPadded.h:76
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:102
Header file for the Or class template.
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:252
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
Header file for the DenseMatrix base class.
Header file for the Columns type trait.
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:3087
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
decltype(auto) decllow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as lower.
Definition: DMatDeclLowExpr.h:1027
Header file for the IsLower type trait.
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:90
Flag for upper matrices.
Definition: TDMatTSMatMultExpr.h:156
Generic wrapper for the null function.
Definition: Noop.h:58
Header file for the IsTriangular type trait.
Constraints on the storage order of matrix types.
Compile time check for symmetric matrices.This type trait tests whether or not the given template par...
Definition: IsSymmetric.h:85
Header file for the exception macros of the math module.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:264
IfTrue_< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: TDMatTSMatMultExpr.h:265
Header file for the DeclDiag functor.
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: TDMatTSMatMultExpr.h:306
Constraint on the data type.
Header file for all forward declarations for expression class templates.
ElementType_< ResultType > ElementType
Resulting element type.
Definition: TDMatTSMatMultExpr.h:250
ResultType_< MT1 > RT1
Result type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:132
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:108
Compile time check for lower unitriangular matrices.This type trait tests whether or not the given te...
Definition: IsUniLower.h:86
Header file for the conjugate shim.
Compile time check for resizable data types.This type trait tests whether the given data type is a re...
Definition: IsResizable.h:75
Header file for the IsSIMDCombinable type trait.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
Header file for run time assertion macros.
Compile time check for column-major matrix types.This type trait tests whether or not the given templ...
Definition: IsColumnMajorMatrix.h:110
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:249
Flag for symmetric matrices.
Definition: TDMatTSMatMultExpr.h:153
Utility type for generic codes.
ElementType_< RT1 > ET1
Element type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:134
CompositeType_< MT1 > CT1
Composite type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:136
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:154
Header file for the reset shim.
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:296
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:94
decltype(auto) declsym(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as symmetric.
Definition: DMatDeclSymExpr.h:1029
Header file for the isDefault shim.
Compile time check for Hermitian matrices.This type trait tests whether or not the given template par...
Definition: IsHermitian.h:85
Constraints on the storage order of matrix types.
Generic wrapper for the declherm() function.
Definition: DeclHerm.h:58
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:819
Header file for the Noop functor.
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
Header file for the RemoveReference type trait.
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:224
typename T::OppositeType OppositeType_
Alias declaration for nested OppositeType type definitions.The OppositeType_ alias declaration provid...
Definition: Aliases.h:263
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:84
Generic wrapper for the declupp() function.
Definition: DeclUpp.h:58
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
const Type & ReturnType
Return type for expression template evaluations.
Definition: CompressedMatrix.h:3082
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: TDMatTSMatMultExpr.h:354
decltype(auto) declherm(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as Hermitian.
Definition: DMatDeclHermExpr.h:1029
Flag for Hermitian matrices.
Definition: TDMatTSMatMultExpr.h:154
Header file for the IsComputation type trait class.
Header file for the IsBuiltin type trait.
Compile time logical or evaluation.The Or alias declaration performs at compile time a logical or (&#39;&&&#3...
Definition: Or.h:76
Flag for lower matrices.
Definition: TDMatTSMatMultExpr.h:155
Header file for the IntegralConstant class template.
Compile time evaluation of the number of columns of a matrix.The Columns type trait evaluates the num...
Definition: Columns.h:75
Generic wrapper for the decldiag() function.
Definition: DeclDiag.h:58
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatTSMatMultExpr.h:451
Compile time evaluation of the number of rows of a matrix.The Rows type trait evaluates the number of...
Definition: Rows.h:75
Header file for the DeclHerm functor.
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TDMatTSMatMultExpr.h:444
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:600
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:423
Header file for the IsUpper type trait.
If_< IsExpression< MT1 >, const MT1, const MT1 &> LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:256
decltype(auto) conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition: DMatMapExpr.h:1321
Constraint on the data type.
Generic wrapper for the declsym() function.
Definition: DeclSym.h:58
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatTSMatMultExpr.h:253
BLAZE_ALWAYS_INLINE bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition: Matrix.h:742
Header file for the IsResizable type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the Bool class template.
Header file for the DeclSym functor.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:61
Header file for the IsExpression type trait class.
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatTSMatMultExpr.h:434
Header file for the function trace functionality.
OppositeType_< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:248