TDMatTSMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
51 #include <blaze/math/Exception.h>
64 #include <blaze/math/shims/Reset.h>
66 #include <blaze/math/SIMD.h>
86 #include <blaze/math/views/Check.h>
91 #include <blaze/util/Assert.h>
92 #include <blaze/util/EnableIf.h>
94 #include <blaze/util/mpl/And.h>
95 #include <blaze/util/mpl/Bool.h>
96 #include <blaze/util/mpl/If.h>
97 #include <blaze/util/mpl/Or.h>
98 #include <blaze/util/Types.h>
101 
102 
103 namespace blaze {
104 
105 //=================================================================================================
106 //
107 // CLASS TDMATTSMATMULTEXPR
108 //
109 //=================================================================================================
110 
111 //*************************************************************************************************
118 template< typename MT1 // Type of the left-hand side dense matrix
119  , typename MT2 // Type of the right-hand side sparse matrix
120  , bool SF // Symmetry flag
121  , bool HF // Hermitian flag
122  , bool LF // Lower flag
123  , bool UF > // Upper flag
124 class TDMatTSMatMultExpr
125  : public MatMatMultExpr< DenseMatrix< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, true > >
126  , private Computation
127 {
128  private:
129  //**Type definitions****************************************************************************
136  //**********************************************************************************************
137 
138  //**********************************************************************************************
140  enum : bool { evaluateLeft = IsComputation<MT1>::value || RequiresEvaluation<MT1>::value };
141  //**********************************************************************************************
142 
143  //**********************************************************************************************
145  enum : bool { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
146  //**********************************************************************************************
147 
148  //**********************************************************************************************
150  enum : bool {
151  SYM = ( SF && !( HF || LF || UF ) ),
152  HERM = ( HF && !( LF || UF ) ),
153  LOW = ( LF || ( ( SF || HF ) && UF ) ),
154  UPP = ( UF || ( ( SF || HF ) && LF ) )
155  };
156  //**********************************************************************************************
157 
158  //**********************************************************************************************
160 
164  template< typename T1, typename T2, typename T3 >
165  struct IsEvaluationRequired {
166  enum : bool { value = ( evaluateLeft || evaluateRight ) };
167  };
169  //**********************************************************************************************
170 
171  //**********************************************************************************************
173 
176  template< typename T1, typename T2, typename T3 >
177  struct UseVectorizedKernel {
178  enum : bool { value = useOptimizedKernels &&
180  T1::simdEnabled && T2::simdEnabled &&
184  , ElementType_<T3> >::value &&
187  };
189  //**********************************************************************************************
190 
191  //**********************************************************************************************
193 
197  template< typename T1, typename T2, typename T3 >
198  struct UseOptimizedKernel {
199  enum : bool { value = useOptimizedKernels &&
200  !UseVectorizedKernel<T1,T2,T3>::value &&
202  !IsResizable< ElementType_<T1> >::value &&
204  };
206  //**********************************************************************************************
207 
208  //**********************************************************************************************
210 
213  template< typename T1, typename T2, typename T3 >
214  struct UseDefaultKernel {
215  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
216  !UseOptimizedKernel<T1,T2,T3>::value };
217  };
219  //**********************************************************************************************
220 
221  //**********************************************************************************************
223 
226  using ForwardFunctor = IfTrue_< HERM
227  , DeclHerm
228  , IfTrue_< SYM
229  , DeclSym
230  , IfTrue_< LOW
231  , IfTrue_< UPP
232  , DeclDiag
233  , DeclLow >
234  , IfTrue_< UPP
235  , DeclUpp
236  , Noop > > > >;
238  //**********************************************************************************************
239 
240  public:
241  //**Type definitions****************************************************************************
244 
250  using ReturnType = const ElementType;
251  using CompositeType = const ResultType;
252 
254  using LeftOperand = If_< IsExpression<MT1>, const MT1, const MT1& >;
255 
257  using RightOperand = If_< IsExpression<MT2>, const MT2, const MT2& >;
258 
261 
264  //**********************************************************************************************
265 
266  //**Compilation flags***************************************************************************
268  enum : bool { simdEnabled = !IsDiagonal<MT1>::value &&
269  MT1::simdEnabled &&
272 
274  enum : bool { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
275  !evaluateRight && MT2::smpAssignable };
276  //**********************************************************************************************
277 
278  //**SIMD properties*****************************************************************************
280  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
281  //**********************************************************************************************
282 
283  //**Constructor*********************************************************************************
289  explicit inline TDMatTSMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
290  : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
291  , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
292  {
293  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
294  }
295  //**********************************************************************************************
296 
297  //**Access operator*****************************************************************************
304  inline ReturnType operator()( size_t i, size_t j ) const {
305  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
306  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
307 
308  if( IsDiagonal<MT1>::value ) {
309  return lhs_(i,i) * rhs_(i,j);
310  }
311  else if( IsDiagonal<MT2>::value ) {
312  return lhs_(i,j) * rhs_(j,j);
313  }
315  const size_t begin( ( IsUpper<MT1>::value )
316  ?( ( IsLower<MT2>::value )
317  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
318  , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
319  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
320  :( ( IsLower<MT2>::value )
321  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
322  :( 0UL ) ) );
323  const size_t end( ( IsLower<MT1>::value )
324  ?( ( IsUpper<MT2>::value )
325  ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
326  , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
327  :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
328  :( ( IsUpper<MT2>::value )
329  ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
330  :( lhs_.columns() ) ) );
331 
332  if( begin >= end ) return ElementType();
333 
334  const size_t n( end - begin );
335 
336  return subvector( row( lhs_, i, unchecked ), begin, n, unchecked ) *
337  subvector( column( rhs_, j, unchecked ), begin, n, unchecked );
338  }
339  else {
340  return row( lhs_, i, unchecked ) * column( rhs_, j, unchecked );
341  }
342  }
343  //**********************************************************************************************
344 
345  //**At function*********************************************************************************
353  inline ReturnType at( size_t i, size_t j ) const {
354  if( i >= lhs_.rows() ) {
355  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
356  }
357  if( j >= rhs_.columns() ) {
358  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
359  }
360  return (*this)(i,j);
361  }
362  //**********************************************************************************************
363 
364  //**Rows function*******************************************************************************
369  inline size_t rows() const noexcept {
370  return lhs_.rows();
371  }
372  //**********************************************************************************************
373 
374  //**Columns function****************************************************************************
379  inline size_t columns() const noexcept {
380  return rhs_.columns();
381  }
382  //**********************************************************************************************
383 
384  //**Left operand access*************************************************************************
389  inline LeftOperand leftOperand() const noexcept {
390  return lhs_;
391  }
392  //**********************************************************************************************
393 
394  //**Right operand access************************************************************************
399  inline RightOperand rightOperand() const noexcept {
400  return rhs_;
401  }
402  //**********************************************************************************************
403 
404  //**********************************************************************************************
410  template< typename T >
411  inline bool canAlias( const T* alias ) const noexcept {
412  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
413  }
414  //**********************************************************************************************
415 
416  //**********************************************************************************************
422  template< typename T >
423  inline bool isAliased( const T* alias ) const noexcept {
424  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
425  }
426  //**********************************************************************************************
427 
428  //**********************************************************************************************
433  inline bool isAligned() const noexcept {
434  return lhs_.isAligned();
435  }
436  //**********************************************************************************************
437 
438  //**********************************************************************************************
443  inline bool canSMPAssign() const noexcept {
444  return ( rows() * columns() >= SMP_TDMATTSMATMULT_THRESHOLD ) && !IsDiagonal<MT1>::value;
445  }
446  //**********************************************************************************************
447 
448  private:
449  //**Member variables****************************************************************************
452  //**********************************************************************************************
453 
454  //**Assignment to dense matrices****************************************************************
467  template< typename MT // Type of the target dense matrix
468  , bool SO > // Storage order of the target dense matrix
469  friend inline void assign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
470  {
472 
473  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
474  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
475 
476  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
477  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
478 
479  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
480  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
481  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
482  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
483  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
484  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
485 
486  TDMatTSMatMultExpr::selectAssignKernel( ~lhs, A, B );
487  }
489  //**********************************************************************************************
490 
491  //**Default assignment to dense matrices********************************************************
505  template< typename MT3 // Type of the left-hand side target matrix
506  , typename MT4 // Type of the left-hand side matrix operand
507  , typename MT5 > // Type of the right-hand side matrix operand
509  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
510  {
512 
513  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
514 
515  reset( C );
516 
517  for( size_t ii=0UL; ii<A.rows(); ii+=block )
518  {
519  const size_t itmp( min( ii+block, A.rows() ) );
520 
521  for( size_t j=0UL; j<B.columns(); ++j )
522  {
523  ConstIterator element( B.begin(j) );
524  const ConstIterator end( B.end(j) );
525 
526  for( ; element!=end; ++element )
527  {
528  const size_t j1( element->index() );
529 
531  {
532  C(j1,j) = A(j1,j1) * element->value();
533  }
534  else
535  {
536  const size_t ibegin( ( IsLower<MT4>::value )
537  ?( ( LOW )
538  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
539  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
540  :( LOW ? max(j,ii) : ii ) );
541  const size_t iend( ( IsUpper<MT4>::value )
542  ?( ( SYM || HERM || UPP )
543  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
544  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
545  :( SYM || HERM || UPP ? min(j+1UL,itmp) : itmp ) );
546 
547  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
548  continue;
549 
550  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
551 
552  for( size_t i=ibegin; i<iend; ++i ) {
553  if( isDefault( C(i,j) ) )
554  C(i,j) = A(i,j1) * element->value();
555  else
556  C(i,j) += A(i,j1) * element->value();
557  }
558  }
559  }
560  }
561  }
562 
563  if( SYM || HERM ) {
564  for( size_t j=0UL; j<B.columns(); ++j ) {
565  for( size_t i=j+1UL; i<A.rows(); ++i ) {
566  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
567  }
568  }
569  }
570  }
572  //**********************************************************************************************
573 
574  //**Optimized assignment to dense matrices******************************************************
588  template< typename MT3 // Type of the left-hand side target matrix
589  , typename MT4 // Type of the left-hand side matrix operand
590  , typename MT5 > // Type of the right-hand side matrix operand
592  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
593  {
595 
596  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
597 
598  reset( C );
599 
600  for( size_t ii=0UL; ii<A.rows(); ii+=block )
601  {
602  const size_t itmp( min( ii+block, A.rows() ) );
603 
604  for( size_t j=0UL; j<B.columns(); ++j )
605  {
606  const ConstIterator end( B.end(j) );
607  ConstIterator element( B.begin(j) );
608 
609  const size_t nonzeros( B.nonZeros(j) );
610  const size_t kpos( nonzeros & size_t(-4) );
611  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
612 
613  for( size_t k=0UL; k<kpos; k+=4UL )
614  {
615  const size_t j1( element->index() );
616  const ET2 v1( element->value() );
617  ++element;
618  const size_t j2( element->index() );
619  const ET2 v2( element->value() );
620  ++element;
621  const size_t j3( element->index() );
622  const ET2 v3( element->value() );
623  ++element;
624  const size_t j4( element->index() );
625  const ET2 v4( element->value() );
626  ++element;
627 
628  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
629 
630  const size_t ibegin( ( IsLower<MT4>::value )
631  ?( ( LOW )
632  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
633  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
634  :( LOW ? max(j,ii) : ii ) );
635  const size_t iend( ( IsUpper<MT4>::value )
636  ?( ( SYM || HERM || UPP )
637  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
638  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) ) )
639  :( SYM || HERM || UPP ? min(j+1UL,itmp) : itmp ) );
640 
641  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
642  continue;
643 
644  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
645 
646  const size_t inum( iend - ibegin );
647  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
648  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
649 
650  for( size_t i=ibegin; i<ipos; i+=4UL ) {
651  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
652  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
653  C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
654  C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
655  }
656  for( size_t i=ipos; i<iend; ++i ) {
657  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
658  }
659  }
660 
661  for( ; element!=end; ++element )
662  {
663  const size_t j1( element->index() );
664  const ET2 v1( element->value() );
665 
666  const size_t ibegin( ( IsLower<MT4>::value )
667  ?( ( LOW )
668  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
669  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
670  :( LOW ? max(j,ii) : ii ) );
671  const size_t iend( ( IsUpper<MT4>::value )
672  ?( ( SYM || HERM || UPP )
673  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
674  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
675  :( SYM || HERM || UPP ? min(j+1UL,itmp) : itmp ) );
676 
677  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
678  continue;
679 
680  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
681 
682  const size_t inum( iend - ibegin );
683  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
684  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
685 
686  for( size_t i=ibegin; i<ipos; i+=4UL ) {
687  C(i ,j) += A(i ,j1) * v1;
688  C(i+1UL,j) += A(i+1UL,j1) * v1;
689  C(i+2UL,j) += A(i+2UL,j1) * v1;
690  C(i+3UL,j) += A(i+3UL,j1) * v1;
691  }
692  for( size_t i=ipos; i<iend; ++i ) {
693  C(i,j) += A(i,j1) * v1;
694  }
695  }
696  }
697  }
698 
699  if( SYM || HERM ) {
700  for( size_t j=0UL; j<B.columns(); ++j ) {
701  for( size_t i=j+1UL; i<A.rows(); ++i ) {
702  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
703  }
704  }
705  }
706  }
708  //**********************************************************************************************
709 
710  //**Vectorized assignment to column-major dense matrices****************************************
724  template< typename MT3 // Type of the left-hand side target matrix
725  , typename MT4 // Type of the left-hand side matrix operand
726  , typename MT5 > // Type of the right-hand side matrix operand
728  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
729  {
731 
732  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
733 
734  reset( C );
735 
736  for( size_t j=0UL; j<B.columns(); ++j )
737  {
738  const ConstIterator end( B.end(j) );
739  ConstIterator element( B.begin(j) );
740 
741  const size_t nonzeros( B.nonZeros(j) );
742  const size_t kpos( nonzeros & size_t(-4) );
743  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
744 
745  for( size_t k=0UL; k<kpos; k+=4UL )
746  {
747  const size_t j1( element->index() );
748  const ET2 v1( element->value() );
749  ++element;
750  const size_t j2( element->index() );
751  const ET2 v2( element->value() );
752  ++element;
753  const size_t j3( element->index() );
754  const ET2 v3( element->value() );
755  ++element;
756  const size_t j4( element->index() );
757  const ET2 v4( element->value() );
758  ++element;
759 
760  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
761 
762  const SIMDType xmm1( set( v1 ) );
763  const SIMDType xmm2( set( v2 ) );
764  const SIMDType xmm3( set( v3 ) );
765  const SIMDType xmm4( set( v4 ) );
766 
767  const size_t ibegin( ( IsLower<MT4>::value )
769  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
770  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
771  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
772  const size_t iend( ( IsUpper<MT4>::value )
774  ?( SYM || HERM || UPP ? max(j+1UL,j4) : j4 )
775  :( SYM || HERM || UPP ? max(j,j4)+1UL : j4+1UL ) )
776  :( SYM || HERM || UPP ? j+1UL : A.rows() ) );
777  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
778 
779  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
780  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
781 
782  size_t i( ibegin );
783 
784  for( ; i<ipos; i+=SIMDSIZE ) {
785  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
786  }
787  for( ; remainder && i<iend; ++i ) {
788  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
789  }
790  }
791 
792  for( ; element!=end; ++element )
793  {
794  const size_t j1( element->index() );
795  const ET2 v1( element->value() );
796 
797  const SIMDType xmm1( set( v1 ) );
798 
799  const size_t ibegin( ( IsLower<MT4>::value )
801  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
802  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
803  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
804  const size_t iend( ( IsUpper<MT4>::value )
806  ?( SYM || HERM || UPP ? max(j+1UL,j1) : j1 )
807  :( SYM || HERM || UPP ? max(j,j1)+1UL : j1+1UL ) )
808  :( SYM || HERM || UPP ? j+1UL : A.rows() ) );
809  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
810 
811  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
812  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
813 
814  size_t i( ibegin );
815 
816  for( ; i<ipos; i+=SIMDSIZE ) {
817  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 );
818  }
819  for( ; remainder && i<iend; ++i ) {
820  C(i,j) += A(i,j1) * v1;
821  }
822  }
823  }
824 
825  if( SYM || HERM ) {
826  for( size_t j=0UL; j<B.columns(); ++j ) {
827  for( size_t i=j+1UL; i<A.rows(); ++i ) {
828  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
829  }
830  }
831  }
832  }
834  //**********************************************************************************************
835 
836  //**Assignment to sparse matrices***************************************************************
849  template< typename MT // Type of the target sparse matrix
850  , bool SO > // Storage order of the target sparse matrix
851  friend inline void assign( SparseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
852  {
854 
856 
863 
864  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
865  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
866 
867  const ForwardFunctor fwd;
868 
869  const TmpType tmp( serial( rhs ) );
870  assign( ~lhs, fwd( tmp ) );
871  }
873  //**********************************************************************************************
874 
875  //**Addition assignment to dense matrices*******************************************************
888  template< typename MT // Type of the target dense matrix
889  , bool SO > // Storage order of the target dense matrix
890  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
891  {
893 
894  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
895  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
896 
897  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
898  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
899 
900  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
901  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
902  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
903  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
904  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
905  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
906 
907  TDMatTSMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
908  }
910  //**********************************************************************************************
911 
912  //**Default addition assignment to dense matrices***********************************************
926  template< typename MT3 // Type of the left-hand side target matrix
927  , typename MT4 // Type of the left-hand side matrix operand
928  , typename MT5 > // Type of the right-hand side matrix operand
930  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
931  {
933 
934  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
935 
936  for( size_t ii=0UL; ii<A.rows(); ii+=block )
937  {
938  const size_t itmp( min( ii+block, A.rows() ) );
939 
940  for( size_t j=0UL; j<B.columns(); ++j )
941  {
942  ConstIterator element( B.begin(j) );
943  const ConstIterator end( B.end(j) );
944 
945  for( ; element!=end; ++element )
946  {
947  const size_t j1( element->index() );
948 
950  {
951  C(j1,j) += A(j1,j1) * element->value();
952  }
953  else
954  {
955  const size_t ibegin( ( IsLower<MT4>::value )
956  ?( ( LOW )
957  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
958  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
959  :( LOW ? max(j,ii) : ii ) );
960  const size_t iend( ( IsUpper<MT4>::value )
961  ?( ( UPP )
962  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
963  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
964  :( UPP ? min(j+1UL,itmp) : itmp ) );
965 
966  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
967  continue;
968 
969  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
970 
971  const size_t inum( iend - ibegin );
972  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
973  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
974 
975  for( size_t i=ibegin; i<ipos; i+=4UL ) {
976  C(i ,j) += A(i ,j1) * element->value();
977  C(i+1UL,j) += A(i+1UL,j1) * element->value();
978  C(i+2UL,j) += A(i+2UL,j1) * element->value();
979  C(i+3UL,j) += A(i+3UL,j1) * element->value();
980  }
981  for( size_t i=ipos; i<iend; ++i ) {
982  C(i,j) += A(i,j1) * element->value();
983  }
984  }
985  }
986  }
987  }
988  }
990  //**********************************************************************************************
991 
992  //**Optimized addition assignment to dense matrices*********************************************
1006  template< typename MT3 // Type of the left-hand side target matrix
1007  , typename MT4 // Type of the left-hand side matrix operand
1008  , typename MT5 > // Type of the right-hand side matrix operand
1010  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1011  {
1013 
1014  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
1015 
1016  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1017  {
1018  const size_t itmp( min( ii+block, A.rows() ) );
1019 
1020  for( size_t j=0UL; j<B.columns(); ++j )
1021  {
1022  const ConstIterator end( B.end(j) );
1023  ConstIterator element( B.begin(j) );
1024 
1025  const size_t nonzeros( B.nonZeros(j) );
1026  const size_t kpos( nonzeros & size_t(-4) );
1027  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1028 
1029  for( size_t k=0UL; k<kpos; k+=4UL )
1030  {
1031  const size_t j1( element->index() );
1032  const ET2 v1( element->value() );
1033  ++element;
1034  const size_t j2( element->index() );
1035  const ET2 v2( element->value() );
1036  ++element;
1037  const size_t j3( element->index() );
1038  const ET2 v3( element->value() );
1039  ++element;
1040  const size_t j4( element->index() );
1041  const ET2 v4( element->value() );
1042  ++element;
1043 
1044  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1045 
1046  const size_t ibegin( ( IsLower<MT4>::value )
1047  ?( ( LOW )
1048  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1049  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1050  :( LOW ? max(j,ii) : ii ) );
1051  const size_t iend( ( IsUpper<MT4>::value )
1052  ?( ( UPP )
1053  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
1054  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) ) )
1055  :( UPP ? min(j+1UL,itmp) : itmp ) );
1056 
1057  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1058  continue;
1059 
1060  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1061 
1062  const size_t inum( iend - ibegin );
1063  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1064  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1065 
1066  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1067  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1068  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1069  C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1070  C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1071  }
1072  for( size_t i=ipos; i<iend; ++i ) {
1073  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1074  }
1075  }
1076 
1077  for( ; element!=end; ++element )
1078  {
1079  const size_t j1( element->index() );
1080  const ET2 v1( element->value() );
1081 
1082  const size_t ibegin( ( IsLower<MT4>::value )
1083  ?( ( LOW )
1084  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1085  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1086  :( LOW ? max(j,ii) : ii ) );
1087  const size_t iend( ( IsUpper<MT4>::value )
1088  ?( ( UPP )
1089  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1090  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
1091  :( UPP ? min(j+1UL,itmp) : itmp ) );
1092 
1093  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1094  continue;
1095 
1096  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1097 
1098  const size_t inum( iend - ibegin );
1099  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1100  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1101 
1102  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1103  C(i ,j) += A(i ,j1) * v1;
1104  C(i+1UL,j) += A(i+1UL,j1) * v1;
1105  C(i+2UL,j) += A(i+2UL,j1) * v1;
1106  C(i+3UL,j) += A(i+3UL,j1) * v1;
1107  }
1108  for( size_t i=ipos; i<iend; ++i ) {
1109  C(i,j) += A(i,j1) * v1;
1110  }
1111  }
1112  }
1113  }
1114  }
1116  //**********************************************************************************************
1117 
1118  //**Vectorized addition assignment to column-major dense matrices*******************************
1132  template< typename MT3 // Type of the left-hand side target matrix
1133  , typename MT4 // Type of the left-hand side matrix operand
1134  , typename MT5 > // Type of the right-hand side matrix operand
1136  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1137  {
1139 
1140  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
1141 
1142  for( size_t j=0UL; j<B.columns(); ++j )
1143  {
1144  const ConstIterator end( B.end(j) );
1145  ConstIterator element( B.begin(j) );
1146 
1147  const size_t nonzeros( B.nonZeros(j) );
1148  const size_t kpos( nonzeros & size_t(-4) );
1149  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1150 
1151  for( size_t k=0UL; k<kpos; k+=4UL )
1152  {
1153  const size_t j1( element->index() );
1154  const ET2 v1( element->value() );
1155  ++element;
1156  const size_t j2( element->index() );
1157  const ET2 v2( element->value() );
1158  ++element;
1159  const size_t j3( element->index() );
1160  const ET2 v3( element->value() );
1161  ++element;
1162  const size_t j4( element->index() );
1163  const ET2 v4( element->value() );
1164  ++element;
1165 
1166  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1167 
1168  const SIMDType xmm1( set( v1 ) );
1169  const SIMDType xmm2( set( v2 ) );
1170  const SIMDType xmm3( set( v3 ) );
1171  const SIMDType xmm4( set( v4 ) );
1172 
1173  const size_t ibegin( ( IsLower<MT4>::value )
1175  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
1176  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
1177  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
1178  const size_t iend( ( IsUpper<MT4>::value )
1180  ?( UPP ? max(j+1UL,j4) : j4 )
1181  :( UPP ? max(j,j4)+1UL : j4+1UL ) )
1182  :( UPP ? j+1UL : A.rows() ) );
1183  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1184 
1185  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1186  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1187 
1188  size_t i( ibegin );
1189 
1190  for( ; i<ipos; i+=SIMDSIZE ) {
1191  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
1192  }
1193  for( ; remainder && i<iend; ++i ) {
1194  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1195  }
1196  }
1197 
1198  for( ; element!=end; ++element )
1199  {
1200  const size_t j1( element->index() );
1201  const ET2 v1( element->value() );
1202 
1203  const SIMDType xmm1( set( v1 ) );
1204 
1205  const size_t ibegin( ( IsLower<MT4>::value )
1207  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
1208  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
1209  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
1210  const size_t iend( ( IsUpper<MT4>::value )
1212  ?( UPP ? max(j+1UL,j1) : j1 )
1213  :( UPP ? max(j,j1)+1UL : j1+1UL ) )
1214  :( UPP ? j+1UL : A.rows() ) );
1215  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1216 
1217  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1218  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1219 
1220  size_t i( ibegin );
1221 
1222  for( ; i<ipos; i+=SIMDSIZE ) {
1223  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 );
1224  }
1225  for( ; remainder && i<iend; ++i ) {
1226  C(i,j) += A(i,j1) * v1;
1227  }
1228  }
1229  }
1230  }
1232  //**********************************************************************************************
1233 
1234  //**Addition assignment to sparse matrices******************************************************
1235  // No special implementation for the addition assignment to sparse matrices.
1236  //**********************************************************************************************
1237 
1238  //**Subtraction assignment to dense matrices****************************************************
1251  template< typename MT // Type of the target dense matrix
1252  , bool SO > // Storage order of the target dense matrix
1253  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1254  {
1256 
1257  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1258  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1259 
1260  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1261  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1262 
1263  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1264  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1265  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1266  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1267  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1268  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1269 
1270  TDMatTSMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1271  }
1273  //**********************************************************************************************
1274 
1275  //**Default subtraction assignment to dense matrices********************************************
1289  template< typename MT3 // Type of the left-hand side target matrix
1290  , typename MT4 // Type of the left-hand side matrix operand
1291  , typename MT5 > // Type of the right-hand side matrix operand
1293  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1294  {
1296 
1297  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
1298 
1299  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1300  {
1301  const size_t itmp( min( ii+block, A.rows() ) );
1302 
1303  for( size_t j=0UL; j<B.columns(); ++j )
1304  {
1305  ConstIterator element( B.begin(j) );
1306  const ConstIterator end( B.end(j) );
1307 
1308  for( ; element!=end; ++element )
1309  {
1310  const size_t j1( element->index() );
1311 
1313  {
1314  C(j1,j) -= A(j1,j1) * element->value();
1315  }
1316  else
1317  {
1318  const size_t ibegin( ( IsLower<MT4>::value )
1319  ?( ( LOW )
1320  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1321  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1322  :( LOW ? max(j,ii) : ii ) );
1323  const size_t iend( ( IsUpper<MT4>::value )
1324  ?( ( UPP )
1325  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1326  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
1327  :( UPP ? min(j+1UL,itmp) : itmp ) );
1328 
1329  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1330  continue;
1331 
1332  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1333 
1334  const size_t inum( iend - ibegin );
1335  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1336  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1337 
1338  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1339  C(i ,j) -= A(i ,j1) * element->value();
1340  C(i+1UL,j) -= A(i+1UL,j1) * element->value();
1341  C(i+2UL,j) -= A(i+2UL,j1) * element->value();
1342  C(i+3UL,j) -= A(i+3UL,j1) * element->value();
1343  }
1344  for( size_t i=ipos; i<iend; ++i ) {
1345  C(i,j) -= A(i,j1) * element->value();
1346  }
1347  }
1348  }
1349  }
1350  }
1351  }
1353  //**********************************************************************************************
1354 
1355  //**Optimized subtraction assignment to dense matrices******************************************
1369  template< typename MT3 // Type of the left-hand side target matrix
1370  , typename MT4 // Type of the left-hand side matrix operand
1371  , typename MT5 > // Type of the right-hand side matrix operand
1373  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1374  {
1376 
1377  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
1378 
1379  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1380  {
1381  const size_t itmp( min( ii+block, A.rows() ) );
1382 
1383  for( size_t j=0UL; j<B.columns(); ++j )
1384  {
1385  const ConstIterator end( B.end(j) );
1386  ConstIterator element( B.begin(j) );
1387 
1388  const size_t nonzeros( B.nonZeros(j) );
1389  const size_t kpos( nonzeros & size_t(-4) );
1390  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1391 
1392  for( size_t k=0UL; k<kpos; k+=4UL )
1393  {
1394  const size_t j1( element->index() );
1395  const ET2 v1( element->value() );
1396  ++element;
1397  const size_t j2( element->index() );
1398  const ET2 v2( element->value() );
1399  ++element;
1400  const size_t j3( element->index() );
1401  const ET2 v3( element->value() );
1402  ++element;
1403  const size_t j4( element->index() );
1404  const ET2 v4( element->value() );
1405  ++element;
1406 
1407  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1408 
1409  const size_t ibegin( ( IsLower<MT4>::value )
1410  ?( ( LOW )
1411  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1412  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1413  :( LOW ? max(j,ii) : ii ) );
1414  const size_t iend( ( IsUpper<MT4>::value )
1415  ?( ( UPP )
1416  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
1417  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) ) )
1418  :( UPP ? min(j+1UL,itmp) : itmp ) );
1419 
1420  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1421  continue;
1422 
1423  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1424 
1425  const size_t inum( iend - ibegin );
1426  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1427  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1428 
1429  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1430  C(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1431  C(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1432  C(i+2UL,j) -= A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1433  C(i+3UL,j) -= A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1434  }
1435  for( size_t i=ipos; i<iend; ++i ) {
1436  C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1437  }
1438  }
1439 
1440  for( ; element!=end; ++element )
1441  {
1442  const size_t j1( element->index() );
1443  const ET2 v1( element->value() );
1444 
1445  const size_t ibegin( ( IsLower<MT4>::value )
1446  ?( ( LOW )
1447  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1448  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1449  :( LOW ? max(j,ii) : ii ) );
1450  const size_t iend( ( IsUpper<MT4>::value )
1451  ?( ( UPP )
1452  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1453  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
1454  :( UPP ? min(j+1UL,itmp) : itmp ) );
1455 
1456  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1457  continue;
1458 
1459  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1460 
1461  const size_t inum( iend - ibegin );
1462  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1463  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1464 
1465  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1466  C(i ,j) -= A(i ,j1) * v1;
1467  C(i+1UL,j) -= A(i+1UL,j1) * v1;
1468  C(i+2UL,j) -= A(i+2UL,j1) * v1;
1469  C(i+3UL,j) -= A(i+3UL,j1) * v1;
1470  }
1471  for( size_t i=ipos; i<iend; ++i ) {
1472  C(i,j) -= A(i,j1) * v1;
1473  }
1474  }
1475  }
1476  }
1477  }
1479  //**********************************************************************************************
1480 
1481  //**Vectorized subtraction assignment to column-major dense matrices****************************
1495  template< typename MT3 // Type of the left-hand side target matrix
1496  , typename MT4 // Type of the left-hand side matrix operand
1497  , typename MT5 > // Type of the right-hand side matrix operand
1499  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1500  {
1502 
1503  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
1504 
1505  for( size_t j=0UL; j<B.columns(); ++j )
1506  {
1507  const ConstIterator end( B.end(j) );
1508  ConstIterator element( B.begin(j) );
1509 
1510  const size_t nonzeros( B.nonZeros(j) );
1511  const size_t kpos( nonzeros & size_t(-4) );
1512  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1513 
1514  for( size_t k=0UL; k<kpos; k+=4UL )
1515  {
1516  const size_t j1( element->index() );
1517  const ET2 v1( element->value() );
1518  ++element;
1519  const size_t j2( element->index() );
1520  const ET2 v2( element->value() );
1521  ++element;
1522  const size_t j3( element->index() );
1523  const ET2 v3( element->value() );
1524  ++element;
1525  const size_t j4( element->index() );
1526  const ET2 v4( element->value() );
1527  ++element;
1528 
1529  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1530 
1531  const SIMDType xmm1( set( v1 ) );
1532  const SIMDType xmm2( set( v2 ) );
1533  const SIMDType xmm3( set( v3 ) );
1534  const SIMDType xmm4( set( v4 ) );
1535 
1536  const size_t ibegin( ( IsLower<MT4>::value )
1538  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
1539  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
1540  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
1541  const size_t iend( ( IsUpper<MT4>::value )
1543  ?( UPP ? max(j+1UL,j4) : j4 )
1544  :( UPP ? max(j,j4)+1UL : j4+1UL ) )
1545  :( UPP ? j+1UL : A.rows() ) );
1546  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1547 
1548  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1549  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1550 
1551  size_t i( ibegin );
1552 
1553  for( ; i<ipos; i+=SIMDSIZE ) {
1554  C.store( i, j, C.load(i,j) - A.load(i,j1) * xmm1 - A.load(i,j2) * xmm2 - A.load(i,j3) * xmm3 - A.load(i,j4) * xmm4 );
1555  }
1556  for( ; remainder && i<iend; ++i ) {
1557  C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1558  }
1559  }
1560 
1561  for( ; element!=end; ++element )
1562  {
1563  const size_t j1( element->index() );
1564  const ET2 v1( element->value() );
1565 
1566  const SIMDType xmm1( set( v1 ) );
1567 
1568  const size_t ibegin( ( IsLower<MT4>::value )
1570  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
1571  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
1572  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
1573  const size_t iend( ( IsUpper<MT4>::value )
1575  ?( UPP ? max(j+1UL,j1) : j1 )
1576  :( UPP ? max(j,j1)+1UL : j1+1UL ) )
1577  :( UPP ? j+1UL : A.rows() ) );
1578  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1579 
1580  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1581  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1582 
1583  size_t i( ibegin );
1584 
1585  for( ; i<ipos; i+=SIMDSIZE ) {
1586  C.store( i, j, C.load(i,j) - A.load(i,j1) * xmm1 );
1587  }
1588  for( ; remainder && i<iend; ++i ) {
1589  C(i,j) -= A(i,j1) * v1;
1590  }
1591  }
1592  }
1593  }
1595  //**********************************************************************************************
1596 
1597  //**Subtraction assignment to sparse matrices***************************************************
1598  // No special implementation for the subtraction assignment to sparse matrices.
1599  //**********************************************************************************************
1600 
1601  //**Schur product assignment to dense matrices**************************************************
1614  template< typename MT // Type of the target dense matrix
1615  , bool SO > // Storage order of the target dense matrix
1616  friend inline void schurAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1617  {
1619 
1623 
1624  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1625  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1626 
1627  const ResultType tmp( serial( rhs ) );
1628  schurAssign( ~lhs, tmp );
1629  }
1631  //**********************************************************************************************
1632 
1633  //**Schur product assignment to sparse matrices*************************************************
1634  // No special implementation for the Schur product assignment to sparse matrices.
1635  //**********************************************************************************************
1636 
1637  //**Multiplication assignment to dense matrices*************************************************
1638  // No special implementation for the multiplication assignment to dense matrices.
1639  //**********************************************************************************************
1640 
1641  //**Multiplication assignment to sparse matrices************************************************
1642  // No special implementation for the multiplication assignment to sparse matrices.
1643  //**********************************************************************************************
1644 
1645  //**SMP assignment to dense matrices************************************************************
1660  template< typename MT // Type of the target dense matrix
1661  , bool SO > // Storage order of the target dense matrix
1663  smpAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1664  {
1666 
1667  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1668  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1669 
1670  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1671  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1672 
1673  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1674  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1675  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1676  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1677  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1678  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1679 
1680  smpAssign( ~lhs, A * B );
1681  }
1683  //**********************************************************************************************
1684 
1685  //**SMP assignment to sparse matrices***********************************************************
1700  template< typename MT // Type of the target sparse matrix
1701  , bool SO > // Storage order of the target sparse matrix
1704  {
1706 
1708 
1715 
1716  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1717  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1718 
1719  const ForwardFunctor fwd;
1720 
1721  const TmpType tmp( rhs );
1722  smpAssign( ~lhs, fwd( tmp ) );
1723  }
1725  //**********************************************************************************************
1726 
1727  //**SMP addition assignment to dense matrices***************************************************
1742  template< typename MT // Type of the target dense matrix
1743  , bool SO > // Storage order of the target dense matrix
1746  {
1748 
1749  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1750  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1751 
1752  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1753  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1754 
1755  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1756  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1757  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1758  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1759  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1760  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1761 
1762  smpAddAssign( ~lhs, A * B );
1763  }
1765  //**********************************************************************************************
1766 
1767  //**SMP addition assignment to sparse matrices**************************************************
1768  // No special implementation for the SMP addition assignment to sparse matrices.
1769  //**********************************************************************************************
1770 
1771  //**SMP subtraction assignment to dense matrices************************************************
1786  template< typename MT // Type of the target dense matrix
1787  , bool SO > // Storage order of the target dense matrix
1790  {
1792 
1793  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1794  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1795 
1796  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1797  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1798 
1799  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1800  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1801  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1802  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1803  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1804  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1805 
1806  smpSubAssign( ~lhs, A * B );
1807  }
1809  //**********************************************************************************************
1810 
1811  //**SMP subtraction assignment to sparse matrices***********************************************
1812  // No special implementation for the SMP subtraction assignment to sparse matrices.
1813  //**********************************************************************************************
1814 
1815  //**SMP Schur product assignment to dense matrices**********************************************
1828  template< typename MT // Type of the target dense matrix
1829  , bool SO > // Storage order of the target dense matrix
1830  friend inline void smpSchurAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1831  {
1833 
1837 
1838  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1839  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1840 
1841  const ResultType tmp( rhs );
1842  smpSchurAssign( ~lhs, tmp );
1843  }
1845  //**********************************************************************************************
1846 
1847  //**SMP Schur product assignment to sparse matrices*********************************************
1848  // No special implementation for the SMP Schur product assignment to sparse matrices.
1849  //**********************************************************************************************
1850 
1851  //**SMP multiplication assignment to dense matrices*********************************************
1852  // No special implementation for the SMP multiplication assignment to dense matrices.
1853  //**********************************************************************************************
1854 
1855  //**SMP multiplication assignment to sparse matrices********************************************
1856  // No special implementation for the SMP multiplication assignment to sparse matrices.
1857  //**********************************************************************************************
1858 
1859  //**Compile time checks*************************************************************************
1867  //**********************************************************************************************
1868 };
1869 //*************************************************************************************************
1870 
1871 
1872 
1873 
1874 //=================================================================================================
1875 //
1876 // GLOBAL BINARY ARITHMETIC OPERATORS
1877 //
1878 //=================================================================================================
1879 
1880 //*************************************************************************************************
1909 template< typename MT1 // Type of the left-hand side dense matrix
1910  , typename MT2 > // Type of the right-hand side sparse matrix
1911 inline decltype(auto)
1912  operator*( const DenseMatrix<MT1,true>& lhs, const SparseMatrix<MT2,true>& rhs )
1913 {
1915 
1916  if( (~lhs).columns() != (~rhs).rows() ) {
1917  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1918  }
1919 
1921  return ReturnType( ~lhs, ~rhs );
1922 }
1923 //*************************************************************************************************
1924 
1925 
1926 
1927 
1928 //=================================================================================================
1929 //
1930 // GLOBAL FUNCTIONS
1931 //
1932 //=================================================================================================
1933 
1934 //*************************************************************************************************
1958 template< typename MT1 // Type of the left-hand side dense matrix
1959  , typename MT2 // Type of the right-hand side dense matrix
1960  , bool SF // Symmetry flag
1961  , bool HF // Hermitian flag
1962  , bool LF // Lower flag
1963  , bool UF > // Upper flag
1964 inline decltype(auto) declsym( const TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
1965 {
1967 
1968  if( !isSquare( dm ) ) {
1969  BLAZE_THROW_INVALID_ARGUMENT( "Invalid symmetric matrix specification" );
1970  }
1971 
1973  return ReturnType( dm.leftOperand(), dm.rightOperand() );
1974 }
1976 //*************************************************************************************************
1977 
1978 
1979 //*************************************************************************************************
2003 template< typename MT1 // Type of the left-hand side dense matrix
2004  , typename MT2 // Type of the right-hand side dense matrix
2005  , bool SF // Symmetry flag
2006  , bool HF // Hermitian flag
2007  , bool LF // Lower flag
2008  , bool UF > // Upper flag
2009 inline decltype(auto) declherm( const TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2010 {
2012 
2013  if( !isSquare( dm ) ) {
2014  BLAZE_THROW_INVALID_ARGUMENT( "Invalid Hermitian matrix specification" );
2015  }
2016 
2018  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2019 }
2021 //*************************************************************************************************
2022 
2023 
2024 //*************************************************************************************************
2048 template< typename MT1 // Type of the left-hand side dense matrix
2049  , typename MT2 // Type of the right-hand side dense matrix
2050  , bool SF // Symmetry flag
2051  , bool HF // Hermitian flag
2052  , bool LF // Lower flag
2053  , bool UF > // Upper flag
2054 inline decltype(auto) decllow( const TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2055 {
2057 
2058  if( !isSquare( dm ) ) {
2059  BLAZE_THROW_INVALID_ARGUMENT( "Invalid lower matrix specification" );
2060  }
2061 
2063  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2064 }
2066 //*************************************************************************************************
2067 
2068 
2069 //*************************************************************************************************
2093 template< typename MT1 // Type of the left-hand side dense matrix
2094  , typename MT2 // Type of the right-hand side dense matrix
2095  , bool SF // Symmetry flag
2096  , bool HF // Hermitian flag
2097  , bool LF // Lower flag
2098  , bool UF > // Upper flag
2099 inline decltype(auto) declupp( const TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2100 {
2102 
2103  if( !isSquare( dm ) ) {
2104  BLAZE_THROW_INVALID_ARGUMENT( "Invalid upper matrix specification" );
2105  }
2106 
2108  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2109 }
2111 //*************************************************************************************************
2112 
2113 
2114 //*************************************************************************************************
2138 template< typename MT1 // Type of the left-hand side dense matrix
2139  , typename MT2 // Type of the right-hand side dense matrix
2140  , bool SF // Symmetry flag
2141  , bool HF // Hermitian flag
2142  , bool LF // Lower flag
2143  , bool UF > // Upper flag
2144 inline decltype(auto) decldiag( const TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2145 {
2147 
2148  if( !isSquare( dm ) ) {
2149  BLAZE_THROW_INVALID_ARGUMENT( "Invalid diagonal matrix specification" );
2150  }
2151 
2153  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2154 }
2156 //*************************************************************************************************
2157 
2158 
2159 
2160 
2161 //=================================================================================================
2162 //
2163 // SIZE SPECIALIZATIONS
2164 //
2165 //=================================================================================================
2166 
2167 //*************************************************************************************************
2169 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2170 struct Size< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 0UL >
2171  : public Size<MT1,0UL>
2172 {};
2173 
2174 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2175 struct Size< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 1UL >
2176  : public Size<MT2,1UL>
2177 {};
2179 //*************************************************************************************************
2180 
2181 
2182 
2183 
2184 //=================================================================================================
2185 //
2186 // ISALIGNED SPECIALIZATIONS
2187 //
2188 //=================================================================================================
2189 
2190 //*************************************************************************************************
2192 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2193 struct IsAligned< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2194  : public IsAligned<MT1>
2195 {};
2197 //*************************************************************************************************
2198 
2199 
2200 
2201 
2202 //=================================================================================================
2203 //
2204 // ISSYMMETRIC SPECIALIZATIONS
2205 //
2206 //=================================================================================================
2207 
2208 //*************************************************************************************************
2210 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2211 struct IsSymmetric< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2212  : public Or< Bool<SF>
2213  , And< Bool<HF>
2214  , IsBuiltin< ElementType_< TDMatTSMatMultExpr<MT1,MT2,false,true,false,false> > > >
2215  , And< Bool<LF>, Bool<UF> > >
2216 {};
2218 //*************************************************************************************************
2219 
2220 
2221 
2222 
2223 //=================================================================================================
2224 //
2225 // ISHERMITIAN SPECIALIZATIONS
2226 //
2227 //=================================================================================================
2228 
2229 //*************************************************************************************************
2231 template< typename MT1, typename MT2, bool SF, bool LF, bool UF >
2232 struct IsHermitian< TDMatTSMatMultExpr<MT1,MT2,SF,true,LF,UF> >
2233  : public TrueType
2234 {};
2236 //*************************************************************************************************
2237 
2238 
2239 
2240 
2241 //=================================================================================================
2242 //
2243 // ISLOWER SPECIALIZATIONS
2244 //
2245 //=================================================================================================
2246 
2247 //*************************************************************************************************
2249 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2250 struct IsLower< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2251  : public Or< Bool<LF>
2252  , And< IsLower<MT1>, IsLower<MT2> >
2253  , And< Or< Bool<SF>, Bool<HF> >
2254  , IsUpper<MT1>, IsUpper<MT2> > >
2255 {};
2257 //*************************************************************************************************
2258 
2259 
2260 
2261 
2262 //=================================================================================================
2263 //
2264 // ISUNILOWER SPECIALIZATIONS
2265 //
2266 //=================================================================================================
2267 
2268 //*************************************************************************************************
2270 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2271 struct IsUniLower< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2272  : public Or< And< IsUniLower<MT1>, IsUniLower<MT2> >
2273  , And< Or< Bool<SF>, Bool<HF> >
2274  , IsUniUpper<MT1>, IsUniUpper<MT2> > >
2275 {};
2277 //*************************************************************************************************
2278 
2279 
2280 
2281 
2282 //=================================================================================================
2283 //
2284 // ISSTRICTLYLOWER SPECIALIZATIONS
2285 //
2286 //=================================================================================================
2287 
2288 //*************************************************************************************************
2290 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2291 struct IsStrictlyLower< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2292  : public Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2293  , And< IsStrictlyLower<MT2>, IsLower<MT1> >
2294  , And< Or< Bool<SF>, Bool<HF> >
2295  , Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2296  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > > > >
2297 {};
2299 //*************************************************************************************************
2300 
2301 
2302 
2303 
2304 //=================================================================================================
2305 //
2306 // ISUPPER SPECIALIZATIONS
2307 //
2308 //=================================================================================================
2309 
2310 //*************************************************************************************************
2312 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2313 struct IsUpper< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2314  : public Or< Bool<UF>
2315  , And< IsUpper<MT1>, IsUpper<MT2> >
2316  , And< Or< Bool<SF>, Bool<HF> >
2317  , IsLower<MT1>, IsLower<MT2> > >
2318 {};
2320 //*************************************************************************************************
2321 
2322 
2323 
2324 
2325 //=================================================================================================
2326 //
2327 // ISUNIUPPER SPECIALIZATIONS
2328 //
2329 //=================================================================================================
2330 
2331 //*************************************************************************************************
2333 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2334 struct IsUniUpper< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2335  : public Or< And< IsUniUpper<MT1>, IsUniUpper<MT2> >
2336  , And< Or< Bool<SF>, Bool<HF> >
2337  , IsUniLower<MT1>, IsUniLower<MT2> > >
2338 {};
2340 //*************************************************************************************************
2341 
2342 
2343 
2344 
2345 //=================================================================================================
2346 //
2347 // ISSTRICTLYUPPER SPECIALIZATIONS
2348 //
2349 //=================================================================================================
2350 
2351 //*************************************************************************************************
2353 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2354 struct IsStrictlyUpper< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2355  : public Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2356  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> >
2357  , And< Or< Bool<SF>, Bool<HF> >
2358  , Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2359  , And< IsStrictlyLower<MT2>, IsLower<MT1> > > > >
2360 {};
2362 //*************************************************************************************************
2363 
2364 } // namespace blaze
2365 
2366 #endif
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:329
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Header file for auxiliary alias declarations.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:131
Headerfile for the generic min algorithm.
Header file for the blaze::checked and blaze::unchecked instances.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:71
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: TDMatTSMatMultExpr.h:451
RightOperand rightOperand() const noexcept
Returns the right-hand side transpose sparse matrix operand.
Definition: TDMatTSMatMultExpr.h:399
decltype(auto) decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as diagonal.
Definition: DMatDeclDiagExpr.h:996
Flag for symmetric matrices.
Definition: TDMatTSMatMultExpr.h:151
Header file for the IsUniUpper type trait.
EnableIf_< IsDenseMatrix< MT1 > > smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:196
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:86
Header file for basic type definitions.
Flag for lower matrices.
Definition: TDMatTSMatMultExpr.h:153
Flag for Hermitian matrices.
Definition: TDMatTSMatMultExpr.h:152
Expression object for transpose dense matrix-transpose sparse matrix multiplications.The TDMatTSMatMultExpr class represents the compile time expression for multiplications between a column-major dense matrix and a column-major sparse matrix.
Definition: Forward.h:155
LeftOperand leftOperand() const noexcept
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatTSMatMultExpr.h:389
EnableIf_< IsDenseMatrix< MT1 > > smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:164
IfTrue_< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatTSMatMultExpr.h:260
Header file for the serial shim.
Header file for the IsDiagonal type trait.
Generic wrapper for a compile time constant integral value.The IntegralConstant class template repres...
Definition: IntegralConstant.h:71
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
CompositeType_< MT2 > CT2
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:135
Header file for the DeclUpp functor.
ElementType_< RT2 > ET2
Element type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:133
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:364
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:172
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:316
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TDMatTSMatMultExpr.h:423
Header file for the IsColumnMajorMatrix type trait.
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:588
constexpr Unchecked unchecked
Global Unchecked instance.The blaze::unchecked instance is an optional token for the creation of view...
Definition: Check.h:138
Header file for the And class template.
const ElementType_< MT > min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1903
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:87
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:171
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TDMatTSMatMultExpr.h:411
Flag for upper matrices.
Definition: TDMatTSMatMultExpr.h:154
decltype(auto) declupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as upper.
Definition: DMatDeclUppExpr.h:1026
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:291
Header file for the Computation base class.
If_< IsExpression< MT2 >, const MT2, const MT2 &> RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:257
Header file for the MatMatMultExpr base class.
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:87
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Compile time check for data types.This type trait tests whether or not the given types can be combine...
Definition: IsSIMDCombinable.h:120
Header file for the IsUniLower type trait.
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:343
const ElementType_< MT > max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1950
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: TDMatTSMatMultExpr.h:249
EnableIf_< IsDenseMatrix< MT1 > > smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:133
MultTrait_< RT1, RT2 > ResultType
Result type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:245
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:80
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:129
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
ResultType_< MT2 > RT2
Result type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:131
Compile time check for the alignment of data types.This type trait tests whether the given data type ...
Definition: IsAligned.h:87
Constraint on the data type.
Constraint on the data type.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:71
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
Compile time check for upper unitriangular matrices.This type trait tests whether or not the given te...
Definition: IsUniUpper.h:86
Headerfile for the generic max algorithm.
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: TDMatTSMatMultExpr.h:379
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: TDMatTSMatMultExpr.h:369
Namespace of the Blaze C++ math library.
Definition: Blaze.h:58
Header file for the DeclLow functor.
Header file for the If class template.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
TDMatTSMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the TDMatTSMatMultExpr class.
Definition: TDMatTSMatMultExpr.h:289
Generic wrapper for the decllow() function.
Definition: DeclLow.h:58
Compile time check for data types with padding.This type trait tests whether the given data type empl...
Definition: IsPadded.h:76
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:102
Header file for the Or class template.
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:250
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
Header file for the DenseMatrix base class.
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:3085
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
decltype(auto) decllow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as lower.
Definition: DMatDeclLowExpr.h:1026
Header file for the IsLower type trait.
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:89
Generic wrapper for the null function.
Definition: Noop.h:59
Header file for the IsTriangular type trait.
Constraints on the storage order of matrix types.
Compile time check for symmetric matrices.This type trait tests whether or not the given template par...
Definition: IsSymmetric.h:85
Header file for the exception macros of the math module.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:430
IfTrue_< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: TDMatTSMatMultExpr.h:263
Header file for the DeclDiag functor.
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: TDMatTSMatMultExpr.h:304
Constraint on the data type.
Header file for all forward declarations for expression class templates.
ElementType_< ResultType > ElementType
Resulting element type.
Definition: TDMatTSMatMultExpr.h:248
ResultType_< MT1 > RT1
Result type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:130
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:107
Compile time check for lower unitriangular matrices.This type trait tests whether or not the given te...
Definition: IsUniLower.h:86
Header file for the conjugate shim.
Compile time check for resizable data types.This type trait tests whether the given data type is a re...
Definition: IsResizable.h:75
Header file for the IsSIMDCombinable type trait.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
Header file for run time assertion macros.
Compile time check for column-major matrix types.This type trait tests whether or not the given templ...
Definition: IsColumnMajorMatrix.h:110
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:247
ElementType_< RT1 > ET1
Element type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:132
CompositeType_< MT1 > CT1
Composite type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:134
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:154
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:131
Header file for the reset shim.
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:296
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:94
decltype(auto) declsym(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as symmetric.
Definition: DMatDeclSymExpr.h:1028
Header file for the isDefault shim.
Compile time check for Hermitian matrices.This type trait tests whether or not the given template par...
Definition: IsHermitian.h:85
Constraints on the storage order of matrix types.
Generic wrapper for the declherm() function.
Definition: DeclHerm.h:58
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:816
Header file for the Noop functor.
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
Header file for the RemoveReference type trait.
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:224
typename T::OppositeType OppositeType_
Alias declaration for nested OppositeType type definitions.The OppositeType_ alias declaration provid...
Definition: Aliases.h:263
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:84
Generic wrapper for the declupp() function.
Definition: DeclUpp.h:58
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
const Type & ReturnType
Return type for expression template evaluations.
Definition: CompressedMatrix.h:3080
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: TDMatTSMatMultExpr.h:353
decltype(auto) declherm(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as Hermitian.
Definition: DMatDeclHermExpr.h:1028
Header file for the IsComputation type trait class.
Header file for the IsBuiltin type trait.
Compile time logical &#39;or&#39; evaluation.The Or alias declaration performs at compile time a logical &#39;or&#39;...
Definition: Or.h:76
Compile time evaluation of the size of vectors and matrices.The Size type trait evaluates the size of...
Definition: Size.h:80
Generic wrapper for the decldiag() function.
Definition: DeclDiag.h:58
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatTSMatMultExpr.h:450
Header file for the DeclHerm functor.
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TDMatTSMatMultExpr.h:443
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:628
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:423
Header file for the IsUpper type trait.
If_< IsExpression< MT1 >, const MT1, const MT1 &> LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:254
decltype(auto) conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition: DMatMapExpr.h:1321
Constraint on the data type.
Generic wrapper for the declsym() function.
Definition: DeclSym.h:58
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatTSMatMultExpr.h:251
BLAZE_ALWAYS_INLINE bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition: Matrix.h:908
Header file for the IsResizable type trait.
Header file for the Size type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the Bool class template.
Header file for the DeclSym functor.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:61
Header file for the IsExpression type trait class.
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatTSMatMultExpr.h:433
Header file for the function trace functionality.
OppositeType_< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:246