TDMatTSMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
50 #include <blaze/math/Exception.h>
55 #include <blaze/math/Functions.h>
64 #include <blaze/math/shims/Reset.h>
66 #include <blaze/math/SIMD.h>
110 #include <blaze/system/Thresholds.h>
111 #include <blaze/util/Assert.h>
113 #include <blaze/util/EnableIf.h>
116 #include <blaze/util/InvalidType.h>
117 #include <blaze/util/mpl/And.h>
118 #include <blaze/util/mpl/Bool.h>
119 #include <blaze/util/mpl/If.h>
120 #include <blaze/util/mpl/Or.h>
121 #include <blaze/util/Types.h>
124 
125 
126 namespace blaze {
127 
128 //=================================================================================================
129 //
130 // CLASS TDMATTSMATMULTEXPR
131 //
132 //=================================================================================================
133 
134 //*************************************************************************************************
141 template< typename MT1 // Type of the left-hand side dense matrix
142  , typename MT2 // Type of the right-hand side sparse matrix
143  , bool SF // Symmetry flag
144  , bool HF // Hermitian flag
145  , bool LF // Lower flag
146  , bool UF > // Upper flag
147 class TDMatTSMatMultExpr : public DenseMatrix< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, true >
148  , private MatMatMultExpr
149  , private Computation
150 {
151  private:
152  //**Type definitions****************************************************************************
159  //**********************************************************************************************
160 
161  //**********************************************************************************************
163  enum : bool { evaluateLeft = IsComputation<MT1>::value || RequiresEvaluation<MT1>::value };
164  //**********************************************************************************************
165 
166  //**********************************************************************************************
168  enum : bool { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
169  //**********************************************************************************************
170 
171  //**********************************************************************************************
173  enum : bool {
174  SYM = ( SF && !( HF || LF || UF ) ),
175  HERM = ( HF && !( LF || UF ) ),
176  LOW = ( LF || ( ( SF || HF ) && UF ) ),
177  UPP = ( UF || ( ( SF || HF ) && LF ) )
178  };
179  //**********************************************************************************************
180 
181  //**********************************************************************************************
183 
187  template< typename T1, typename T2, typename T3 >
188  struct IsEvaluationRequired {
189  enum : bool { value = ( evaluateLeft || evaluateRight ) };
190  };
192  //**********************************************************************************************
193 
194  //**********************************************************************************************
196 
199  template< typename T1, typename T2, typename T3 >
200  struct UseVectorizedKernel {
201  enum : bool { value = useOptimizedKernels &&
203  T1::simdEnabled && T2::simdEnabled &&
207  , ElementType_<T3> >::value &&
210  };
212  //**********************************************************************************************
213 
214  //**********************************************************************************************
216 
220  template< typename T1, typename T2, typename T3 >
221  struct UseOptimizedKernel {
222  enum : bool { value = useOptimizedKernels &&
223  !UseVectorizedKernel<T1,T2,T3>::value &&
225  !IsResizable< ElementType_<T1> >::value &&
227  };
229  //**********************************************************************************************
230 
231  //**********************************************************************************************
233 
236  template< typename T1, typename T2, typename T3 >
237  struct UseDefaultKernel {
238  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
239  !UseOptimizedKernel<T1,T2,T3>::value };
240  };
242  //**********************************************************************************************
243 
244  //**********************************************************************************************
246 
249  typedef IfTrue_< HERM
250  , DeclHerm
251  , IfTrue_< SYM
252  , DeclSym
253  , IfTrue_< LOW
254  , IfTrue_< UPP
255  , DeclDiag
256  , DeclLow >
257  , IfTrue_< UPP
258  , DeclUpp
259  , Noop > > > > ForwardFunctor;
261  //**********************************************************************************************
262 
263  public:
264  //**Type definitions****************************************************************************
267 
273  typedef const ElementType ReturnType;
274  typedef const ResultType CompositeType;
275 
277  typedef If_< IsExpression<MT1>, const MT1, const MT1& > LeftOperand;
278 
280  typedef If_< IsExpression<MT2>, const MT2, const MT2& > RightOperand;
281 
284 
287  //**********************************************************************************************
288 
289  //**Compilation flags***************************************************************************
291  enum : bool { simdEnabled = !IsDiagonal<MT1>::value &&
292  MT1::simdEnabled &&
295 
297  enum : bool { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
298  !evaluateRight && MT2::smpAssignable };
299  //**********************************************************************************************
300 
301  //**SIMD properties*****************************************************************************
303  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
304  //**********************************************************************************************
305 
306  //**Constructor*********************************************************************************
312  explicit inline TDMatTSMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
313  : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
314  , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
315  {
316  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
317  }
318  //**********************************************************************************************
319 
320  //**Access operator*****************************************************************************
327  inline ReturnType operator()( size_t i, size_t j ) const {
328  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
329  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
330 
331  if( IsDiagonal<MT1>::value ) {
332  return lhs_(i,i) * rhs_(i,j);
333  }
334  else if( IsDiagonal<MT2>::value ) {
335  return lhs_(i,j) * rhs_(j,j);
336  }
338  const size_t begin( ( IsUpper<MT1>::value )
339  ?( ( IsLower<MT2>::value )
340  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
341  , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
342  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
343  :( ( IsLower<MT2>::value )
344  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
345  :( 0UL ) ) );
346  const size_t end( ( IsLower<MT1>::value )
347  ?( ( IsUpper<MT2>::value )
348  ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
349  , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
350  :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
351  :( ( IsUpper<MT2>::value )
352  ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
353  :( lhs_.columns() ) ) );
354 
355  if( begin >= end ) return ElementType();
356 
357  const size_t n( end - begin );
358 
359  return subvector( row( lhs_, i ), begin, n ) * subvector( column( rhs_, j ), begin, n );
360  }
361  else {
362  return row( lhs_, i ) * column( rhs_, j );
363  }
364  }
365  //**********************************************************************************************
366 
367  //**At function*********************************************************************************
375  inline ReturnType at( size_t i, size_t j ) const {
376  if( i >= lhs_.rows() ) {
377  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
378  }
379  if( j >= rhs_.columns() ) {
380  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
381  }
382  return (*this)(i,j);
383  }
384  //**********************************************************************************************
385 
386  //**Rows function*******************************************************************************
391  inline size_t rows() const noexcept {
392  return lhs_.rows();
393  }
394  //**********************************************************************************************
395 
396  //**Columns function****************************************************************************
401  inline size_t columns() const noexcept {
402  return rhs_.columns();
403  }
404  //**********************************************************************************************
405 
406  //**Left operand access*************************************************************************
411  inline LeftOperand leftOperand() const noexcept {
412  return lhs_;
413  }
414  //**********************************************************************************************
415 
416  //**Right operand access************************************************************************
421  inline RightOperand rightOperand() const noexcept {
422  return rhs_;
423  }
424  //**********************************************************************************************
425 
426  //**********************************************************************************************
432  template< typename T >
433  inline bool canAlias( const T* alias ) const noexcept {
434  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
435  }
436  //**********************************************************************************************
437 
438  //**********************************************************************************************
444  template< typename T >
445  inline bool isAliased( const T* alias ) const noexcept {
446  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
447  }
448  //**********************************************************************************************
449 
450  //**********************************************************************************************
455  inline bool isAligned() const noexcept {
456  return lhs_.isAligned();
457  }
458  //**********************************************************************************************
459 
460  //**********************************************************************************************
465  inline bool canSMPAssign() const noexcept {
466  return ( rows() * columns() >= SMP_TDMATTSMATMULT_THRESHOLD ) && !IsDiagonal<MT1>::value;
467  }
468  //**********************************************************************************************
469 
470  private:
471  //**Member variables****************************************************************************
472  LeftOperand lhs_;
473  RightOperand rhs_;
474  //**********************************************************************************************
475 
476  //**Assignment to dense matrices****************************************************************
489  template< typename MT // Type of the target dense matrix
490  , bool SO > // Storage order of the target dense matrix
491  friend inline void assign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
492  {
494 
495  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
496  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
497 
498  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
499  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
500 
501  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
502  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
503  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
504  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
505  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
506  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
507 
508  TDMatTSMatMultExpr::selectAssignKernel( ~lhs, A, B );
509  }
511  //**********************************************************************************************
512 
513  //**Default assignment to dense matrices********************************************************
527  template< typename MT3 // Type of the left-hand side target matrix
528  , typename MT4 // Type of the left-hand side matrix operand
529  , typename MT5 > // Type of the right-hand side matrix operand
531  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
532  {
534 
535  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
536 
537  reset( C );
538 
539  for( size_t ii=0UL; ii<A.rows(); ii+=block )
540  {
541  const size_t itmp( min( ii+block, A.rows() ) );
542 
543  for( size_t j=0UL; j<B.columns(); ++j )
544  {
545  ConstIterator element( B.begin(j) );
546  const ConstIterator end( B.end(j) );
547 
548  for( ; element!=end; ++element )
549  {
550  const size_t j1( element->index() );
551 
553  {
554  C(j1,j) = A(j1,j1) * element->value();
555  }
556  else
557  {
558  const size_t ibegin( ( IsLower<MT4>::value )
559  ?( ( LOW )
560  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
561  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
562  :( LOW ? max(j,ii) : ii ) );
563  const size_t iend( ( IsUpper<MT4>::value )
564  ?( ( SYM || HERM || UPP )
565  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
566  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
567  :( SYM || HERM || UPP ? min(j+1UL,itmp) : itmp ) );
568 
569  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
570  continue;
571 
572  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
573 
574  for( size_t i=ibegin; i<iend; ++i ) {
575  if( isDefault( C(i,j) ) )
576  C(i,j) = A(i,j1) * element->value();
577  else
578  C(i,j) += A(i,j1) * element->value();
579  }
580  }
581  }
582  }
583  }
584 
585  if( SYM || HERM ) {
586  for( size_t j=0UL; j<B.columns(); ++j ) {
587  for( size_t i=j+1UL; i<A.rows(); ++i ) {
588  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
589  }
590  }
591  }
592  }
594  //**********************************************************************************************
595 
596  //**Optimized assignment to dense matrices******************************************************
610  template< typename MT3 // Type of the left-hand side target matrix
611  , typename MT4 // Type of the left-hand side matrix operand
612  , typename MT5 > // Type of the right-hand side matrix operand
614  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
615  {
617 
618  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
619 
620  reset( C );
621 
622  for( size_t ii=0UL; ii<A.rows(); ii+=block )
623  {
624  const size_t itmp( min( ii+block, A.rows() ) );
625 
626  for( size_t j=0UL; j<B.columns(); ++j )
627  {
628  const ConstIterator end( B.end(j) );
629  ConstIterator element( B.begin(j) );
630 
631  const size_t nonzeros( B.nonZeros(j) );
632  const size_t kpos( nonzeros & size_t(-4) );
633  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
634 
635  for( size_t k=0UL; k<kpos; k+=4UL )
636  {
637  const size_t j1( element->index() );
638  const ET2 v1( element->value() );
639  ++element;
640  const size_t j2( element->index() );
641  const ET2 v2( element->value() );
642  ++element;
643  const size_t j3( element->index() );
644  const ET2 v3( element->value() );
645  ++element;
646  const size_t j4( element->index() );
647  const ET2 v4( element->value() );
648  ++element;
649 
650  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
651 
652  const size_t ibegin( ( IsLower<MT4>::value )
653  ?( ( LOW )
654  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
655  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
656  :( LOW ? max(j,ii) : ii ) );
657  const size_t iend( ( IsUpper<MT4>::value )
658  ?( ( SYM || HERM || UPP )
659  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
660  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) ) )
661  :( SYM || HERM || UPP ? min(j+1UL,itmp) : itmp ) );
662 
663  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
664  continue;
665 
666  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
667 
668  const size_t inum( iend - ibegin );
669  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
670  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
671 
672  for( size_t i=ibegin; i<ipos; i+=4UL ) {
673  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
674  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
675  C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
676  C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
677  }
678  for( size_t i=ipos; i<iend; ++i ) {
679  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
680  }
681  }
682 
683  for( ; element!=end; ++element )
684  {
685  const size_t j1( element->index() );
686  const ET2 v1( element->value() );
687 
688  const size_t ibegin( ( IsLower<MT4>::value )
689  ?( ( LOW )
690  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
691  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
692  :( LOW ? max(j,ii) : ii ) );
693  const size_t iend( ( IsUpper<MT4>::value )
694  ?( ( SYM || HERM || UPP )
695  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
696  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
697  :( SYM || HERM || UPP ? min(j+1UL,itmp) : itmp ) );
698 
699  if( ( SYM || HERM || LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
700  continue;
701 
702  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
703 
704  const size_t inum( iend - ibegin );
705  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
706  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
707 
708  for( size_t i=ibegin; i<ipos; i+=4UL ) {
709  C(i ,j) += A(i ,j1) * v1;
710  C(i+1UL,j) += A(i+1UL,j1) * v1;
711  C(i+2UL,j) += A(i+2UL,j1) * v1;
712  C(i+3UL,j) += A(i+3UL,j1) * v1;
713  }
714  for( size_t i=ipos; i<iend; ++i ) {
715  C(i,j) += A(i,j1) * v1;
716  }
717  }
718  }
719  }
720 
721  if( SYM || HERM ) {
722  for( size_t j=0UL; j<B.columns(); ++j ) {
723  for( size_t i=j+1UL; i<A.rows(); ++i ) {
724  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
725  }
726  }
727  }
728  }
730  //**********************************************************************************************
731 
732  //**Vectorized assignment to column-major dense matrices****************************************
746  template< typename MT3 // Type of the left-hand side target matrix
747  , typename MT4 // Type of the left-hand side matrix operand
748  , typename MT5 > // Type of the right-hand side matrix operand
750  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
751  {
753 
754  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
755 
756  reset( C );
757 
758  for( size_t j=0UL; j<B.columns(); ++j )
759  {
760  const ConstIterator end( B.end(j) );
761  ConstIterator element( B.begin(j) );
762 
763  const size_t nonzeros( B.nonZeros(j) );
764  const size_t kpos( nonzeros & size_t(-4) );
765  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
766 
767  for( size_t k=0UL; k<kpos; k+=4UL )
768  {
769  const size_t j1( element->index() );
770  const ET2 v1( element->value() );
771  ++element;
772  const size_t j2( element->index() );
773  const ET2 v2( element->value() );
774  ++element;
775  const size_t j3( element->index() );
776  const ET2 v3( element->value() );
777  ++element;
778  const size_t j4( element->index() );
779  const ET2 v4( element->value() );
780  ++element;
781 
782  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
783 
784  const SIMDType xmm1( set( v1 ) );
785  const SIMDType xmm2( set( v2 ) );
786  const SIMDType xmm3( set( v3 ) );
787  const SIMDType xmm4( set( v4 ) );
788 
789  const size_t ibegin( ( IsLower<MT4>::value )
791  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
792  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
793  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
794  const size_t iend( ( IsUpper<MT4>::value )
796  ?( SYM || HERM || UPP ? max(j+1UL,j4) : j4 )
797  :( SYM || HERM || UPP ? max(j,j4)+1UL : j4+1UL ) )
798  :( SYM || HERM || UPP ? j+1UL : A.rows() ) );
799  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
800 
801  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
802  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
803 
804  size_t i( ibegin );
805 
806  for( ; i<ipos; i+=SIMDSIZE ) {
807  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
808  }
809  for( ; remainder && i<iend; ++i ) {
810  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
811  }
812  }
813 
814  for( ; element!=end; ++element )
815  {
816  const size_t j1( element->index() );
817  const ET2 v1( element->value() );
818 
819  const SIMDType xmm1( set( v1 ) );
820 
821  const size_t ibegin( ( IsLower<MT4>::value )
823  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
824  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
825  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
826  const size_t iend( ( IsUpper<MT4>::value )
828  ?( SYM || HERM || UPP ? max(j+1UL,j1) : j1 )
829  :( SYM || HERM || UPP ? max(j,j1)+1UL : j1+1UL ) )
830  :( SYM || HERM || UPP ? j+1UL : A.rows() ) );
831  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
832 
833  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
834  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
835 
836  size_t i( ibegin );
837 
838  for( ; i<ipos; i+=SIMDSIZE ) {
839  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 );
840  }
841  for( ; remainder && i<iend; ++i ) {
842  C(i,j) += A(i,j1) * v1;
843  }
844  }
845  }
846 
847  if( SYM || HERM ) {
848  for( size_t j=0UL; j<B.columns(); ++j ) {
849  for( size_t i=j+1UL; i<A.rows(); ++i ) {
850  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
851  }
852  }
853  }
854  }
856  //**********************************************************************************************
857 
858  //**Assignment to sparse matrices***************************************************************
871  template< typename MT // Type of the target sparse matrix
872  , bool SO > // Storage order of the target sparse matrix
873  friend inline void assign( SparseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
874  {
876 
878 
885 
886  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
887  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
888 
889  const ForwardFunctor fwd;
890 
891  const TmpType tmp( serial( rhs ) );
892  assign( ~lhs, fwd( tmp ) );
893  }
895  //**********************************************************************************************
896 
897  //**Addition assignment to dense matrices*******************************************************
910  template< typename MT // Type of the target dense matrix
911  , bool SO > // Storage order of the target dense matrix
912  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
913  {
915 
916  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
917  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
918 
919  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
920  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
921 
922  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
923  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
924  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
925  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
926  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
927  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
928 
929  TDMatTSMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
930  }
932  //**********************************************************************************************
933 
934  //**Default addition assignment to dense matrices***********************************************
948  template< typename MT3 // Type of the left-hand side target matrix
949  , typename MT4 // Type of the left-hand side matrix operand
950  , typename MT5 > // Type of the right-hand side matrix operand
952  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
953  {
955 
956  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
957 
958  for( size_t ii=0UL; ii<A.rows(); ii+=block )
959  {
960  const size_t itmp( min( ii+block, A.rows() ) );
961 
962  for( size_t j=0UL; j<B.columns(); ++j )
963  {
964  ConstIterator element( B.begin(j) );
965  const ConstIterator end( B.end(j) );
966 
967  for( ; element!=end; ++element )
968  {
969  const size_t j1( element->index() );
970 
972  {
973  C(j1,j) += A(j1,j1) * element->value();
974  }
975  else
976  {
977  const size_t ibegin( ( IsLower<MT4>::value )
978  ?( ( LOW )
979  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
980  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
981  :( LOW ? max(j,ii) : ii ) );
982  const size_t iend( ( IsUpper<MT4>::value )
983  ?( ( UPP )
984  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
985  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
986  :( UPP ? min(j+1UL,itmp) : itmp ) );
987 
988  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
989  continue;
990 
991  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
992 
993  const size_t inum( iend - ibegin );
994  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
995  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
996 
997  for( size_t i=ibegin; i<ipos; i+=4UL ) {
998  C(i ,j) += A(i ,j1) * element->value();
999  C(i+1UL,j) += A(i+1UL,j1) * element->value();
1000  C(i+2UL,j) += A(i+2UL,j1) * element->value();
1001  C(i+3UL,j) += A(i+3UL,j1) * element->value();
1002  }
1003  for( size_t i=ipos; i<iend; ++i ) {
1004  C(i,j) += A(i,j1) * element->value();
1005  }
1006  }
1007  }
1008  }
1009  }
1010  }
1012  //**********************************************************************************************
1013 
1014  //**Optimized addition assignment to dense matrices*********************************************
1028  template< typename MT3 // Type of the left-hand side target matrix
1029  , typename MT4 // Type of the left-hand side matrix operand
1030  , typename MT5 > // Type of the right-hand side matrix operand
1032  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1033  {
1035 
1036  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
1037 
1038  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1039  {
1040  const size_t itmp( min( ii+block, A.rows() ) );
1041 
1042  for( size_t j=0UL; j<B.columns(); ++j )
1043  {
1044  const ConstIterator end( B.end(j) );
1045  ConstIterator element( B.begin(j) );
1046 
1047  const size_t nonzeros( B.nonZeros(j) );
1048  const size_t kpos( nonzeros & size_t(-4) );
1049  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1050 
1051  for( size_t k=0UL; k<kpos; k+=4UL )
1052  {
1053  const size_t j1( element->index() );
1054  const ET2 v1( element->value() );
1055  ++element;
1056  const size_t j2( element->index() );
1057  const ET2 v2( element->value() );
1058  ++element;
1059  const size_t j3( element->index() );
1060  const ET2 v3( element->value() );
1061  ++element;
1062  const size_t j4( element->index() );
1063  const ET2 v4( element->value() );
1064  ++element;
1065 
1066  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1067 
1068  const size_t ibegin( ( IsLower<MT4>::value )
1069  ?( ( LOW )
1070  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1071  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1072  :( LOW ? max(j,ii) : ii ) );
1073  const size_t iend( ( IsUpper<MT4>::value )
1074  ?( ( UPP )
1075  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
1076  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) ) )
1077  :( UPP ? min(j+1UL,itmp) : itmp ) );
1078 
1079  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1080  continue;
1081 
1082  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1083 
1084  const size_t inum( iend - ibegin );
1085  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1086  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1087 
1088  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1089  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1090  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1091  C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1092  C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1093  }
1094  for( size_t i=ipos; i<iend; ++i ) {
1095  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1096  }
1097  }
1098 
1099  for( ; element!=end; ++element )
1100  {
1101  const size_t j1( element->index() );
1102  const ET2 v1( element->value() );
1103 
1104  const size_t ibegin( ( IsLower<MT4>::value )
1105  ?( ( LOW )
1106  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1107  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1108  :( LOW ? max(j,ii) : ii ) );
1109  const size_t iend( ( IsUpper<MT4>::value )
1110  ?( ( UPP )
1111  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1112  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
1113  :( UPP ? min(j+1UL,itmp) : itmp ) );
1114 
1115  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1116  continue;
1117 
1118  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1119 
1120  const size_t inum( iend - ibegin );
1121  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1122  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1123 
1124  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1125  C(i ,j) += A(i ,j1) * v1;
1126  C(i+1UL,j) += A(i+1UL,j1) * v1;
1127  C(i+2UL,j) += A(i+2UL,j1) * v1;
1128  C(i+3UL,j) += A(i+3UL,j1) * v1;
1129  }
1130  for( size_t i=ipos; i<iend; ++i ) {
1131  C(i,j) += A(i,j1) * v1;
1132  }
1133  }
1134  }
1135  }
1136  }
1138  //**********************************************************************************************
1139 
1140  //**Vectorized addition assignment to column-major dense matrices*******************************
1154  template< typename MT3 // Type of the left-hand side target matrix
1155  , typename MT4 // Type of the left-hand side matrix operand
1156  , typename MT5 > // Type of the right-hand side matrix operand
1158  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1159  {
1161 
1162  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
1163 
1164  for( size_t j=0UL; j<B.columns(); ++j )
1165  {
1166  const ConstIterator end( B.end(j) );
1167  ConstIterator element( B.begin(j) );
1168 
1169  const size_t nonzeros( B.nonZeros(j) );
1170  const size_t kpos( nonzeros & size_t(-4) );
1171  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1172 
1173  for( size_t k=0UL; k<kpos; k+=4UL )
1174  {
1175  const size_t j1( element->index() );
1176  const ET2 v1( element->value() );
1177  ++element;
1178  const size_t j2( element->index() );
1179  const ET2 v2( element->value() );
1180  ++element;
1181  const size_t j3( element->index() );
1182  const ET2 v3( element->value() );
1183  ++element;
1184  const size_t j4( element->index() );
1185  const ET2 v4( element->value() );
1186  ++element;
1187 
1188  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1189 
1190  const SIMDType xmm1( set( v1 ) );
1191  const SIMDType xmm2( set( v2 ) );
1192  const SIMDType xmm3( set( v3 ) );
1193  const SIMDType xmm4( set( v4 ) );
1194 
1195  const size_t ibegin( ( IsLower<MT4>::value )
1197  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
1198  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
1199  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
1200  const size_t iend( ( IsUpper<MT4>::value )
1202  ?( UPP ? max(j+1UL,j4) : j4 )
1203  :( UPP ? max(j,j4)+1UL : j4+1UL ) )
1204  :( UPP ? j+1UL : A.rows() ) );
1205  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1206 
1207  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1208  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1209 
1210  size_t i( ibegin );
1211 
1212  for( ; i<ipos; i+=SIMDSIZE ) {
1213  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
1214  }
1215  for( ; remainder && i<iend; ++i ) {
1216  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1217  }
1218  }
1219 
1220  for( ; element!=end; ++element )
1221  {
1222  const size_t j1( element->index() );
1223  const ET2 v1( element->value() );
1224 
1225  const SIMDType xmm1( set( v1 ) );
1226 
1227  const size_t ibegin( ( IsLower<MT4>::value )
1229  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
1230  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
1231  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
1232  const size_t iend( ( IsUpper<MT4>::value )
1234  ?( UPP ? max(j+1UL,j1) : j1 )
1235  :( UPP ? max(j,j1)+1UL : j1+1UL ) )
1236  :( UPP ? j+1UL : A.rows() ) );
1237  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1238 
1239  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1240  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1241 
1242  size_t i( ibegin );
1243 
1244  for( ; i<ipos; i+=SIMDSIZE ) {
1245  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 );
1246  }
1247  for( ; remainder && i<iend; ++i ) {
1248  C(i,j) += A(i,j1) * v1;
1249  }
1250  }
1251  }
1252  }
1254  //**********************************************************************************************
1255 
1256  //**Addition assignment to sparse matrices******************************************************
1257  // No special implementation for the addition assignment to sparse matrices.
1258  //**********************************************************************************************
1259 
1260  //**Subtraction assignment to dense matrices****************************************************
1273  template< typename MT // Type of the target dense matrix
1274  , bool SO > // Storage order of the target dense matrix
1275  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1276  {
1278 
1279  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1280  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1281 
1282  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1283  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1284 
1285  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1286  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1287  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1288  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1289  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1290  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1291 
1292  TDMatTSMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1293  }
1295  //**********************************************************************************************
1296 
1297  //**Default subtraction assignment to dense matrices********************************************
1311  template< typename MT3 // Type of the left-hand side target matrix
1312  , typename MT4 // Type of the left-hand side matrix operand
1313  , typename MT5 > // Type of the right-hand side matrix operand
1315  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1316  {
1318 
1319  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
1320 
1321  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1322  {
1323  const size_t itmp( min( ii+block, A.rows() ) );
1324 
1325  for( size_t j=0UL; j<B.columns(); ++j )
1326  {
1327  ConstIterator element( B.begin(j) );
1328  const ConstIterator end( B.end(j) );
1329 
1330  for( ; element!=end; ++element )
1331  {
1332  const size_t j1( element->index() );
1333 
1335  {
1336  C(j1,j) -= A(j1,j1) * element->value();
1337  }
1338  else
1339  {
1340  const size_t ibegin( ( IsLower<MT4>::value )
1341  ?( ( LOW )
1342  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1343  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1344  :( LOW ? max(j,ii) : ii ) );
1345  const size_t iend( ( IsUpper<MT4>::value )
1346  ?( ( UPP )
1347  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1348  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
1349  :( UPP ? min(j+1UL,itmp) : itmp ) );
1350 
1351  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1352  continue;
1353 
1354  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1355 
1356  const size_t inum( iend - ibegin );
1357  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1358  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1359 
1360  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1361  C(i ,j) -= A(i ,j1) * element->value();
1362  C(i+1UL,j) -= A(i+1UL,j1) * element->value();
1363  C(i+2UL,j) -= A(i+2UL,j1) * element->value();
1364  C(i+3UL,j) -= A(i+3UL,j1) * element->value();
1365  }
1366  for( size_t i=ipos; i<iend; ++i ) {
1367  C(i,j) -= A(i,j1) * element->value();
1368  }
1369  }
1370  }
1371  }
1372  }
1373  }
1375  //**********************************************************************************************
1376 
1377  //**Optimized subtraction assignment to dense matrices******************************************
1391  template< typename MT3 // Type of the left-hand side target matrix
1392  , typename MT4 // Type of the left-hand side matrix operand
1393  , typename MT5 > // Type of the right-hand side matrix operand
1395  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1396  {
1398 
1399  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
1400 
1401  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1402  {
1403  const size_t itmp( min( ii+block, A.rows() ) );
1404 
1405  for( size_t j=0UL; j<B.columns(); ++j )
1406  {
1407  const ConstIterator end( B.end(j) );
1408  ConstIterator element( B.begin(j) );
1409 
1410  const size_t nonzeros( B.nonZeros(j) );
1411  const size_t kpos( nonzeros & size_t(-4) );
1412  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1413 
1414  for( size_t k=0UL; k<kpos; k+=4UL )
1415  {
1416  const size_t j1( element->index() );
1417  const ET2 v1( element->value() );
1418  ++element;
1419  const size_t j2( element->index() );
1420  const ET2 v2( element->value() );
1421  ++element;
1422  const size_t j3( element->index() );
1423  const ET2 v3( element->value() );
1424  ++element;
1425  const size_t j4( element->index() );
1426  const ET2 v4( element->value() );
1427  ++element;
1428 
1429  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1430 
1431  const size_t ibegin( ( IsLower<MT4>::value )
1432  ?( ( LOW )
1433  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1434  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1435  :( LOW ? max(j,ii) : ii ) );
1436  const size_t iend( ( IsUpper<MT4>::value )
1437  ?( ( UPP )
1438  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
1439  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) ) )
1440  :( UPP ? min(j+1UL,itmp) : itmp ) );
1441 
1442  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1443  continue;
1444 
1445  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1446 
1447  const size_t inum( iend - ibegin );
1448  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1449  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1450 
1451  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1452  C(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1453  C(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1454  C(i+2UL,j) -= A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1455  C(i+3UL,j) -= A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1456  }
1457  for( size_t i=ipos; i<iend; ++i ) {
1458  C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1459  }
1460  }
1461 
1462  for( ; element!=end; ++element )
1463  {
1464  const size_t j1( element->index() );
1465  const ET2 v1( element->value() );
1466 
1467  const size_t ibegin( ( IsLower<MT4>::value )
1468  ?( ( LOW )
1469  ?( max( j, ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1470  :( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) ) )
1471  :( LOW ? max(j,ii) : ii ) );
1472  const size_t iend( ( IsUpper<MT4>::value )
1473  ?( ( UPP )
1474  ?( min( j+1UL, itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1475  :( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) ) )
1476  :( UPP ? min(j+1UL,itmp) : itmp ) );
1477 
1478  if( ( LOW || UPP || IsTriangular<MT4>::value ) && ( ibegin >= iend ) )
1479  continue;
1480 
1481  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1482 
1483  const size_t inum( iend - ibegin );
1484  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1485  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1486 
1487  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1488  C(i ,j) -= A(i ,j1) * v1;
1489  C(i+1UL,j) -= A(i+1UL,j1) * v1;
1490  C(i+2UL,j) -= A(i+2UL,j1) * v1;
1491  C(i+3UL,j) -= A(i+3UL,j1) * v1;
1492  }
1493  for( size_t i=ipos; i<iend; ++i ) {
1494  C(i,j) -= A(i,j1) * v1;
1495  }
1496  }
1497  }
1498  }
1499  }
1501  //**********************************************************************************************
1502 
1503  //**Vectorized subtraction assignment to column-major dense matrices****************************
1517  template< typename MT3 // Type of the left-hand side target matrix
1518  , typename MT4 // Type of the left-hand side matrix operand
1519  , typename MT5 > // Type of the right-hand side matrix operand
1521  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1522  {
1524 
1525  constexpr bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
1526 
1527  for( size_t j=0UL; j<B.columns(); ++j )
1528  {
1529  const ConstIterator end( B.end(j) );
1530  ConstIterator element( B.begin(j) );
1531 
1532  const size_t nonzeros( B.nonZeros(j) );
1533  const size_t kpos( nonzeros & size_t(-4) );
1534  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1535 
1536  for( size_t k=0UL; k<kpos; k+=4UL )
1537  {
1538  const size_t j1( element->index() );
1539  const ET2 v1( element->value() );
1540  ++element;
1541  const size_t j2( element->index() );
1542  const ET2 v2( element->value() );
1543  ++element;
1544  const size_t j3( element->index() );
1545  const ET2 v3( element->value() );
1546  ++element;
1547  const size_t j4( element->index() );
1548  const ET2 v4( element->value() );
1549  ++element;
1550 
1551  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1552 
1553  const SIMDType xmm1( set( v1 ) );
1554  const SIMDType xmm2( set( v2 ) );
1555  const SIMDType xmm3( set( v3 ) );
1556  const SIMDType xmm4( set( v4 ) );
1557 
1558  const size_t ibegin( ( IsLower<MT4>::value )
1560  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
1561  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
1562  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
1563  const size_t iend( ( IsUpper<MT4>::value )
1565  ?( UPP ? max(j+1UL,j4) : j4 )
1566  :( UPP ? max(j,j4)+1UL : j4+1UL ) )
1567  :( UPP ? j+1UL : A.rows() ) );
1568  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1569 
1570  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1571  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1572 
1573  size_t i( ibegin );
1574 
1575  for( ; i<ipos; i+=SIMDSIZE ) {
1576  C.store( i, j, C.load(i,j) - A.load(i,j1) * xmm1 - A.load(i,j2) * xmm2 - A.load(i,j3) * xmm3 - A.load(i,j4) * xmm4 );
1577  }
1578  for( ; remainder && i<iend; ++i ) {
1579  C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1580  }
1581  }
1582 
1583  for( ; element!=end; ++element )
1584  {
1585  const size_t j1( element->index() );
1586  const ET2 v1( element->value() );
1587 
1588  const SIMDType xmm1( set( v1 ) );
1589 
1590  const size_t ibegin( ( IsLower<MT4>::value )
1592  ?( ( LOW ? max(j,j1+1UL) : j1+1UL ) & size_t(-SIMDSIZE) )
1593  :( ( LOW ? max(j,j1) : j1 ) & size_t(-SIMDSIZE) ) )
1594  :( LOW ? ( j & size_t(-SIMDSIZE) ) : 0UL ) );
1595  const size_t iend( ( IsUpper<MT4>::value )
1597  ?( UPP ? max(j+1UL,j1) : j1 )
1598  :( UPP ? max(j,j1)+1UL : j1+1UL ) )
1599  :( UPP ? j+1UL : A.rows() ) );
1600  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1601 
1602  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1603  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1604 
1605  size_t i( ibegin );
1606 
1607  for( ; i<ipos; i+=SIMDSIZE ) {
1608  C.store( i, j, C.load(i,j) - A.load(i,j1) * xmm1 );
1609  }
1610  for( ; remainder && i<iend; ++i ) {
1611  C(i,j) -= A(i,j1) * v1;
1612  }
1613  }
1614  }
1615  }
1617  //**********************************************************************************************
1618 
1619  //**Subtraction assignment to sparse matrices***************************************************
1620  // No special implementation for the subtraction assignment to sparse matrices.
1621  //**********************************************************************************************
1622 
1623  //**Multiplication assignment to dense matrices*************************************************
1624  // No special implementation for the multiplication assignment to dense matrices.
1625  //**********************************************************************************************
1626 
1627  //**Multiplication assignment to sparse matrices************************************************
1628  // No special implementation for the multiplication assignment to sparse matrices.
1629  //**********************************************************************************************
1630 
1631  //**SMP assignment to dense matrices************************************************************
1646  template< typename MT // Type of the target dense matrix
1647  , bool SO > // Storage order of the target dense matrix
1649  smpAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1650  {
1652 
1653  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1654  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1655 
1656  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1657  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1658 
1659  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1660  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1661  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1662  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1663  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1664  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1665 
1666  smpAssign( ~lhs, A * B );
1667  }
1669  //**********************************************************************************************
1670 
1671  //**SMP assignment to sparse matrices***********************************************************
1686  template< typename MT // Type of the target sparse matrix
1687  , bool SO > // Storage order of the target sparse matrix
1690  {
1692 
1694 
1701 
1702  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1703  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1704 
1705  const ForwardFunctor fwd;
1706 
1707  const TmpType tmp( rhs );
1708  smpAssign( ~lhs, fwd( tmp ) );
1709  }
1711  //**********************************************************************************************
1712 
1713  //**SMP addition assignment to dense matrices***************************************************
1728  template< typename MT // Type of the target dense matrix
1729  , bool SO > // Storage order of the target dense matrix
1732  {
1734 
1735  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1736  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1737 
1738  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1739  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1740 
1741  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1742  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1743  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1744  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1745  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1746  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1747 
1748  smpAddAssign( ~lhs, A * B );
1749  }
1751  //**********************************************************************************************
1752 
1753  //**SMP addition assignment to sparse matrices**************************************************
1754  // No special implementation for the SMP addition assignment to sparse matrices.
1755  //**********************************************************************************************
1756 
1757  //**SMP subtraction assignment to dense matrices************************************************
1772  template< typename MT // Type of the target dense matrix
1773  , bool SO > // Storage order of the target dense matrix
1776  {
1778 
1779  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1780  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1781 
1782  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1783  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1784 
1785  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1786  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1787  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1788  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1789  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1790  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1791 
1792  smpSubAssign( ~lhs, A * B );
1793  }
1795  //**********************************************************************************************
1796 
1797  //**SMP subtraction assignment to sparse matrices***********************************************
1798  // No special implementation for the SMP subtraction assignment to sparse matrices.
1799  //**********************************************************************************************
1800 
1801  //**SMP multiplication assignment to dense matrices*********************************************
1802  // No special implementation for the SMP multiplication assignment to dense matrices.
1803  //**********************************************************************************************
1804 
1805  //**SMP multiplication assignment to sparse matrices********************************************
1806  // No special implementation for the SMP multiplication assignment to sparse matrices.
1807  //**********************************************************************************************
1808 
1809  //**Compile time checks*************************************************************************
1817  //**********************************************************************************************
1818 };
1819 //*************************************************************************************************
1820 
1821 
1822 
1823 
1824 //=================================================================================================
1825 //
1826 // GLOBAL BINARY ARITHMETIC OPERATORS
1827 //
1828 //=================================================================================================
1829 
1830 //*************************************************************************************************
1859 template< typename T1 // Type of the left-hand side dense matrix
1860  , typename T2 > // Type of the right-hand side sparse matrix
1863 {
1865 
1866  if( (~lhs).columns() != (~rhs).rows() ) {
1867  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1868  }
1869 
1871 }
1872 //*************************************************************************************************
1873 
1874 
1875 
1876 
1877 //=================================================================================================
1878 //
1879 // GLOBAL FUNCTIONS
1880 //
1881 //=================================================================================================
1882 
1883 //*************************************************************************************************
1907 template< typename MT1 // Type of the left-hand side dense matrix
1908  , typename MT2 // Type of the right-hand side dense matrix
1909  , bool SF // Symmetry flag
1910  , bool HF // Hermitian flag
1911  , bool LF // Lower flag
1912  , bool UF > // Upper flag
1915 {
1917 
1918  if( !isSquare( dm ) ) {
1919  BLAZE_THROW_INVALID_ARGUMENT( "Invalid symmetric matrix specification" );
1920  }
1921 
1922  return TDMatTSMatMultExpr<MT1,MT2,true,HF,LF,UF>( dm.leftOperand(), dm.rightOperand() );
1923 }
1925 //*************************************************************************************************
1926 
1927 
1928 //*************************************************************************************************
1952 template< typename MT1 // Type of the left-hand side dense matrix
1953  , typename MT2 // Type of the right-hand side dense matrix
1954  , bool SF // Symmetry flag
1955  , bool HF // Hermitian flag
1956  , bool LF // Lower flag
1957  , bool UF > // Upper flag
1960 {
1962 
1963  if( !isSquare( dm ) ) {
1964  BLAZE_THROW_INVALID_ARGUMENT( "Invalid Hermitian matrix specification" );
1965  }
1966 
1967  return TDMatTSMatMultExpr<MT1,MT2,SF,true,LF,UF>( dm.leftOperand(), dm.rightOperand() );
1968 }
1970 //*************************************************************************************************
1971 
1972 
1973 //*************************************************************************************************
1997 template< typename MT1 // Type of the left-hand side dense matrix
1998  , typename MT2 // Type of the right-hand side dense matrix
1999  , bool SF // Symmetry flag
2000  , bool HF // Hermitian flag
2001  , bool LF // Lower flag
2002  , bool UF > // Upper flag
2005 {
2007 
2008  if( !isSquare( dm ) ) {
2009  BLAZE_THROW_INVALID_ARGUMENT( "Invalid lower matrix specification" );
2010  }
2011 
2012  return TDMatTSMatMultExpr<MT1,MT2,SF,HF,true,UF>( dm.leftOperand(), dm.rightOperand() );
2013 }
2015 //*************************************************************************************************
2016 
2017 
2018 //*************************************************************************************************
2042 template< typename MT1 // Type of the left-hand side dense matrix
2043  , typename MT2 // Type of the right-hand side dense matrix
2044  , bool SF // Symmetry flag
2045  , bool HF // Hermitian flag
2046  , bool LF // Lower flag
2047  , bool UF > // Upper flag
2050 {
2052 
2053  if( !isSquare( dm ) ) {
2054  BLAZE_THROW_INVALID_ARGUMENT( "Invalid upper matrix specification" );
2055  }
2056 
2057  return TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,true>( dm.leftOperand(), dm.rightOperand() );
2058 }
2060 //*************************************************************************************************
2061 
2062 
2063 //*************************************************************************************************
2087 template< typename MT1 // Type of the left-hand side dense matrix
2088  , typename MT2 // Type of the right-hand side dense matrix
2089  , bool SF // Symmetry flag
2090  , bool HF // Hermitian flag
2091  , bool LF // Lower flag
2092  , bool UF > // Upper flag
2095 {
2097 
2098  if( !isSquare( dm ) ) {
2099  BLAZE_THROW_INVALID_ARGUMENT( "Invalid diagonal matrix specification" );
2100  }
2101 
2102  return TDMatTSMatMultExpr<MT1,MT2,SF,HF,true,true>( dm.leftOperand(), dm.rightOperand() );
2103 }
2105 //*************************************************************************************************
2106 
2107 
2108 
2109 
2110 //=================================================================================================
2111 //
2112 // ROWS SPECIALIZATIONS
2113 //
2114 //=================================================================================================
2115 
2116 //*************************************************************************************************
2118 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2119 struct Rows< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> > : public Rows<MT1>
2120 {};
2122 //*************************************************************************************************
2123 
2124 
2125 
2126 
2127 //=================================================================================================
2128 //
2129 // COLUMNS SPECIALIZATIONS
2130 //
2131 //=================================================================================================
2132 
2133 //*************************************************************************************************
2135 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2136 struct Columns< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> > : public Columns<MT2>
2137 {};
2139 //*************************************************************************************************
2140 
2141 
2142 
2143 
2144 //=================================================================================================
2145 //
2146 // ISALIGNED SPECIALIZATIONS
2147 //
2148 //=================================================================================================
2149 
2150 //*************************************************************************************************
2152 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2153 struct IsAligned< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2154  : public BoolConstant< IsAligned<MT1>::value >
2155 {};
2157 //*************************************************************************************************
2158 
2159 
2160 
2161 
2162 //=================================================================================================
2163 //
2164 // ISSYMMETRIC SPECIALIZATIONS
2165 //
2166 //=================================================================================================
2167 
2168 //*************************************************************************************************
2170 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2171 struct IsSymmetric< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2172  : public BoolConstant< Or< Bool<SF>
2173  , And< Bool<HF>
2174  , IsBuiltin< ElementType_< TDMatTSMatMultExpr<MT1,MT2,false,true,false,false> > > >
2175  , And< Bool<LF>, Bool<UF> > >::value >
2176 {};
2178 //*************************************************************************************************
2179 
2180 
2181 
2182 
2183 //=================================================================================================
2184 //
2185 // ISHERMITIAN SPECIALIZATIONS
2186 //
2187 //=================================================================================================
2188 
2189 //*************************************************************************************************
2191 template< typename MT1, typename MT2, bool SF, bool LF, bool UF >
2192 struct IsHermitian< TDMatTSMatMultExpr<MT1,MT2,SF,true,LF,UF> >
2193  : public TrueType
2194 {};
2196 //*************************************************************************************************
2197 
2198 
2199 
2200 
2201 //=================================================================================================
2202 //
2203 // ISLOWER SPECIALIZATIONS
2204 //
2205 //=================================================================================================
2206 
2207 //*************************************************************************************************
2209 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2210 struct IsLower< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2211  : public BoolConstant< Or< Bool<LF>
2212  , And< IsLower<MT1>, IsLower<MT2> >
2213  , And< Or< Bool<SF>, Bool<HF> >
2214  , IsUpper<MT1>, IsUpper<MT2> > >::value >
2215 {};
2217 //*************************************************************************************************
2218 
2219 
2220 
2221 
2222 //=================================================================================================
2223 //
2224 // ISUNILOWER SPECIALIZATIONS
2225 //
2226 //=================================================================================================
2227 
2228 //*************************************************************************************************
2230 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2231 struct IsUniLower< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2232  : public BoolConstant< Or< And< IsUniLower<MT1>, IsUniLower<MT2> >
2233  , And< Or< Bool<SF>, Bool<HF> >
2234  , IsUniUpper<MT1>, IsUniUpper<MT2> > >::value >
2235 {};
2237 //*************************************************************************************************
2238 
2239 
2240 
2241 
2242 //=================================================================================================
2243 //
2244 // ISSTRICTLYLOWER SPECIALIZATIONS
2245 //
2246 //=================================================================================================
2247 
2248 //*************************************************************************************************
2250 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2251 struct IsStrictlyLower< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2252  : public BoolConstant< Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2253  , And< IsStrictlyLower<MT2>, IsLower<MT1> >
2254  , And< Or< Bool<SF>, Bool<HF> >
2255  , Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2256  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > > > >::value >
2257 {};
2259 //*************************************************************************************************
2260 
2261 
2262 
2263 
2264 //=================================================================================================
2265 //
2266 // ISUPPER SPECIALIZATIONS
2267 //
2268 //=================================================================================================
2269 
2270 //*************************************************************************************************
2272 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2273 struct IsUpper< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2274  : public BoolConstant< Or< Bool<UF>
2275  , And< IsUpper<MT1>, IsUpper<MT2> >
2276  , And< Or< Bool<SF>, Bool<HF> >
2277  , IsLower<MT1>, IsLower<MT2> > >::value >
2278 {};
2280 //*************************************************************************************************
2281 
2282 
2283 
2284 
2285 //=================================================================================================
2286 //
2287 // ISUNIUPPER SPECIALIZATIONS
2288 //
2289 //=================================================================================================
2290 
2291 //*************************************************************************************************
2293 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2294 struct IsUniUpper< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2295  : public BoolConstant< Or< And< IsUniUpper<MT1>, IsUniUpper<MT2> >
2296  , And< Or< Bool<SF>, Bool<HF> >
2297  , IsUniLower<MT1>, IsUniLower<MT2> > >::value >
2298 {};
2300 //*************************************************************************************************
2301 
2302 
2303 
2304 
2305 //=================================================================================================
2306 //
2307 // ISSTRICTLYUPPER SPECIALIZATIONS
2308 //
2309 //=================================================================================================
2310 
2311 //*************************************************************************************************
2313 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2314 struct IsStrictlyUpper< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2315  : public BoolConstant< Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2316  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> >
2317  , And< Or< Bool<SF>, Bool<HF> >
2318  , Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2319  , And< IsStrictlyLower<MT2>, IsLower<MT1> > > > >::value >
2320 {};
2322 //*************************************************************************************************
2323 
2324 
2325 
2326 
2327 //=================================================================================================
2328 //
2329 // EXPRESSION TRAIT SPECIALIZATIONS
2330 //
2331 //=================================================================================================
2332 
2333 //*************************************************************************************************
2335 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF, typename VT >
2336 struct TDMatDVecMultExprTrait< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, VT >
2337 {
2338  public:
2339  //**********************************************************************************************
2344  , INVALID_TYPE >;
2345  //**********************************************************************************************
2346 };
2348 //*************************************************************************************************
2349 
2350 
2351 //*************************************************************************************************
2353 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF, typename VT >
2354 struct TDMatSVecMultExprTrait< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, VT >
2355 {
2356  public:
2357  //**********************************************************************************************
2362  , INVALID_TYPE >;
2363  //**********************************************************************************************
2364 };
2366 //*************************************************************************************************
2367 
2368 
2369 //*************************************************************************************************
2371 template< typename VT, typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2372 struct TDVecTDMatMultExprTrait< VT, TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2373 {
2374  public:
2375  //**********************************************************************************************
2380  , INVALID_TYPE >;
2381  //**********************************************************************************************
2382 };
2384 //*************************************************************************************************
2385 
2386 
2387 //*************************************************************************************************
2389 template< typename VT, typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2390 struct TSVecTDMatMultExprTrait< VT, TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2391 {
2392  public:
2393  //**********************************************************************************************
2398  , INVALID_TYPE >;
2399  //**********************************************************************************************
2400 };
2402 //*************************************************************************************************
2403 
2404 
2405 //*************************************************************************************************
2407 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2408 struct TDMatDeclSymExprTrait< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2409 {
2410  public:
2411  //**********************************************************************************************
2415  , INVALID_TYPE >;
2416  //**********************************************************************************************
2417 };
2419 //*************************************************************************************************
2420 
2421 
2422 //*************************************************************************************************
2424 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2425 struct TDMatDeclHermExprTrait< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2426 {
2427  public:
2428  //**********************************************************************************************
2432  , INVALID_TYPE >;
2433  //**********************************************************************************************
2434 };
2436 //*************************************************************************************************
2437 
2438 
2439 //*************************************************************************************************
2441 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2442 struct TDMatDeclLowExprTrait< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2443 {
2444  public:
2445  //**********************************************************************************************
2449  , INVALID_TYPE >;
2450  //**********************************************************************************************
2451 };
2453 //*************************************************************************************************
2454 
2455 
2456 //*************************************************************************************************
2458 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2459 struct TDMatDeclUppExprTrait< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2460 {
2461  public:
2462  //**********************************************************************************************
2466  , INVALID_TYPE >;
2467  //**********************************************************************************************
2468 };
2470 //*************************************************************************************************
2471 
2472 
2473 //*************************************************************************************************
2475 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2476 struct TDMatDeclDiagExprTrait< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2477 {
2478  public:
2479  //**********************************************************************************************
2483  , INVALID_TYPE >;
2484  //**********************************************************************************************
2485 };
2487 //*************************************************************************************************
2488 
2489 
2490 //*************************************************************************************************
2492 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF, bool AF >
2493 struct SubmatrixExprTrait< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, AF >
2494 {
2495  public:
2496  //**********************************************************************************************
2499  //**********************************************************************************************
2500 };
2502 //*************************************************************************************************
2503 
2504 
2505 //*************************************************************************************************
2507 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2508 struct RowExprTrait< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2509 {
2510  public:
2511  //**********************************************************************************************
2512  using Type = MultExprTrait_< RowExprTrait_<const MT1>, MT2 >;
2513  //**********************************************************************************************
2514 };
2516 //*************************************************************************************************
2517 
2518 
2519 //*************************************************************************************************
2521 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2522 struct ColumnExprTrait< TDMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2523 {
2524  public:
2525  //**********************************************************************************************
2527  //**********************************************************************************************
2528 };
2530 //*************************************************************************************************
2531 
2532 } // namespace blaze
2533 
2534 #endif
typename SubmatrixExprTrait< MT, AF >::Type SubmatrixExprTrait_
Auxiliary alias declaration for the SubmatrixExprTrait type trait.The SubmatrixExprTrait_ alias decla...
Definition: SubmatrixExprTrait.h:134
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Evaluation of the expression type of a dense matrix declherm operation.Via this type trait it is poss...
Definition: TDMatDeclHermExprTrait.h:75
ElementType_< RT1 > ET1
Element type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:155
Compile time check for row vector types.This type trait tests whether or not the given template argum...
Definition: IsRowVector.h:80
const DMatForEachExpr< MT, Conj, SO > conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition: DMatForEachExpr.h:1214
Header file for auxiliary alias declarations.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:72
Evaluation of the expression type of a dense matrix decllow operation.Via this type trait it is possi...
Definition: TDMatDeclLowExprTrait.h:75
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: TDMatTSMatMultExpr.h:473
Header file for mathematical functions.
RightOperand rightOperand() const noexcept
Returns the right-hand side transpose sparse matrix operand.
Definition: TDMatTSMatMultExpr.h:421
constexpr bool useOptimizedKernels
Configuration switch for optimized kernels.This configuration switch enables/disables all optimized c...
Definition: Optimizations.h:84
If_< IsExpression< MT1 >, const MT1, const MT1 &> LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:277
ElementType_< ResultType > ElementType
Resulting element type.
Definition: TDMatTSMatMultExpr.h:271
MultTrait_< RT1, RT2 > ResultType
Result type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:268
Header file for the Rows type trait.
Header file for the IsUniUpper type trait.
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:87
Header file for basic type definitions.
Flag for symmetric matrices.
Definition: TDMatTSMatMultExpr.h:174
Expression object for transpose dense matrix-transpose sparse matrix multiplications.The TDMatTSMatMultExpr class represents the compile time expression for multiplications between a column-major dense matrix and a column-major sparse matrix.
Definition: Forward.h:145
LeftOperand leftOperand() const noexcept
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatTSMatMultExpr.h:411
EnableIf_< IsDenseMatrix< MT1 > > smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
Header file for the IsSparseMatrix type trait.
Header file for the serial shim.
Header file for the IsDiagonal type trait.
Generic wrapper for a compile time constant integral value.The IntegralConstant class template repres...
Definition: IntegralConstant.h:71
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the ColumnExprTrait class template.
Header file for the DeclUpp functor.
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:273
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:194
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:162
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:315
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TDMatTSMatMultExpr.h:445
Header file for the IsColumnMajorMatrix type trait.
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:533
Header file for the IsRowVector type trait.
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:270
Header file for the And class template.
const ElementType_< MT > min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1755
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:88
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:163
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TDMatTSMatMultExpr.h:433
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:721
Evaluation of the expression type of a sparse vector/transpose dense matrix multiplication.Via this type trait it is possible to evaluate the resulting expression type of a sparse vector/transpose dense matrix multiplication. Given the transpose sparse vector type VT and the column-major dense matrix type MT, the nested type Type corresponds to the resulting expression type. In case either VT is not a transpose sparse vector type or MT is not a column-major dense matrix type, the resulting data type Type is set to INVALID_TYPE.
Definition: TSVecTDMatMultExprTrait.h:81
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:245
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
DisableIf_< IsSymmetric< MT >, const DMatDeclSymExpr< MT, SO > > declsym(const DenseMatrix< MT, SO > &dm)
Declares the given non-symmetric dense matrix expression dm as symmetric.
Definition: DMatDeclSymExpr.h:841
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:88
Constraints on the storage order of matrix types.
Header file for the TDMatDeclDiagExprTrait class template.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Compile time check for data types.This type trait tests whether or not the given types can be combine...
Definition: IsSIMDCombinable.h:120
Header file for the IsUniLower type trait.
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:323
const ElementType_< MT > max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1802
EnableIf_< IsDenseMatrix< MT1 > > smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:71
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:119
If_< IsExpression< MT2 >, const MT2, const MT2 &> RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:280
Constraint on the data type.
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
Compile time check for the alignment of data types.This type trait tests whether the given data type ...
Definition: IsAligned.h:87
Constraint on the data type.
Constraint on the data type.
typename MultExprTrait< T1, T2 >::Type MultExprTrait_
Auxiliary alias declaration for the MultExprTrait class template.The MultExprTrait_ alias declaration...
Definition: MultExprTrait.h:344
Header file for the MultExprTrait class template.
DisableIf_< IsHermitian< MT >, const DMatDeclHermExpr< MT, SO > > declherm(const DenseMatrix< MT, SO > &dm)
Declares the given non-Hermitian dense matrix expression dm as Hermitian.
Definition: DMatDeclHermExpr.h:841
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:72
SubvectorExprTrait_< VT, unaligned > subvector(Vector< VT, TF > &vector, size_t index, size_t size)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:152
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
Compile time check for upper unitriangular matrices.This type trait tests whether or not the given te...
Definition: IsUniUpper.h:86
IfTrue_< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatTSMatMultExpr.h:283
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: TDMatTSMatMultExpr.h:401
Compile time check for dense vector types.This type trait tests whether or not the given template par...
Definition: IsDenseVector.h:78
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: TDMatTSMatMultExpr.h:391
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
Header file for the DeclLow functor.
Header file for the If class template.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
Header file for the TSVecTDMatMultExprTrait class template.
OppositeType_< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:269
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2939
TDMatTSMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the TDMatTSMatMultExpr class.
Definition: TDMatTSMatMultExpr.h:312
Evaluation of the expression type of a dense matrix declupp operation.Via this type trait it is possi...
Definition: TDMatDeclUppExprTrait.h:75
Generic wrapper for the decllow() function.
Definition: DeclLow.h:58
Compile time check for data types with padding.This type trait tests whether the given data type empl...
Definition: IsPadded.h:76
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Header file for the Or class template.
Header file for the TDMatSVecMultExprTrait class template.
Header file for the TDVecTSMatMultExprTrait class template.
Header file for the TDMatDeclHermExprTrait class template.
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
Header file for the DenseMatrix base class.
Header file for the Columns type trait.
Header file for the TSMatDVecMultExprTrait class template.
Header file for the TDMatDeclUppExprTrait class template.
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
Evaluation of the expression type of a dense matrix decldiag operation.Via this type trait it is poss...
Definition: TDMatDeclDiagExprTrait.h:75
Compile time check for sparse vector types.This type trait tests whether or not the given template pa...
Definition: IsSparseVector.h:78
Evaluation of the expression type type of a submatrix operation.Via this type trait it is possible to...
Definition: SubmatrixExprTrait.h:80
IfTrue_< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: TDMatTSMatMultExpr.h:286
Header file for the IsLower type trait.
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:90
DisableIf_< Or< IsComputation< MT >, IsTransExpr< MT >, IsDeclExpr< MT > >, RowExprTrait_< MT > > row(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific row of the given matrix.
Definition: Row.h:128
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:60
Generic wrapper for the null function.
Definition: Noop.h:58
Header file for the IsTriangular type trait.
Compile time check for column vector types.This type trait tests whether or not the given template ar...
Definition: IsColumnVector.h:80
Evaluation of the expression type of a dense matrix declsym operation.Via this type trait it is possi...
Definition: TDMatDeclSymExprTrait.h:75
Constraints on the storage order of matrix types.
Compile time check for symmetric matrices.This type trait tests whether or not the given template par...
Definition: IsSymmetric.h:85
Header file for the exception macros of the math module.
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: TDMatTSMatMultExpr.h:272
DisableIf_< IsLower< MT >, const DMatDeclLowExpr< MT, SO > > decllow(const DenseMatrix< MT, SO > &dm)
Declares the given non-lower dense matrix expression dm as lower.
Definition: DMatDeclLowExpr.h:842
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
Evaluation of the expression type type of a row operation.Via this type trait it is possible to evalu...
Definition: RowExprTrait.h:79
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:260
Header file for the DeclDiag functor.
Compile time check for dense matrix types.This type trait tests whether or not the given template par...
Definition: IsDenseMatrix.h:78
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: TDMatTSMatMultExpr.h:327
Header file for the RowExprTrait class template.
Header file for all forward declarations for expression class templates.
Header file for the IsDenseMatrix type trait.
DisableIf_< Or< IsComputation< MT >, IsTransExpr< MT >, IsDeclExpr< MT > >, ColumnExprTrait_< MT > > column(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific column of the given matrix.
Definition: Column.h:128
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
Flag for lower matrices.
Definition: TDMatTSMatMultExpr.h:176
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:109
Compile time check for lower unitriangular matrices.This type trait tests whether or not the given te...
Definition: IsUniLower.h:86
Header file for the conjugate shim.
Compile time check for resizable data types.This type trait tests whether the given data type is a re...
Definition: IsResizable.h:75
Evaluation of the expression type of a transpose dense matrix/sparse vector multiplication.Via this type trait it is possible to evaluate the resulting expression type of a transpose dense matrix/sparse vector multiplication. Given the column-major dense matrix type MT and the non-transpose sparse vector type VT, the nested type Type corresponds to the resulting expression type. In case either MT is not a column-major dense matrix type or VT is not a non-transpose sparse vector type, the resulting data type Type is set to INVALID_TYPE.
Definition: TDMatSVecMultExprTrait.h:79
Header file for the IsSIMDCombinable type trait.
Header file for the IsSparseVector type trait.
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
Header file for run time assertion macros.
Compile time check for column-major matrix types.This type trait tests whether or not the given templ...
Definition: IsColumnMajorMatrix.h:83
Utility type for generic codes.
ElementType_< RT2 > ET2
Element type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:156
Header file for the TDMatDeclLowExprTrait class template.
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:160
TDMatTSMatMultExpr< MT1, MT2, SF, HF, LF, UF > This
Type of this TDMatTSMatMultExpr instance.
Definition: TDMatTSMatMultExpr.h:266
Header file for the reset shim.
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:296
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:93
ResultType_< MT2 > RT2
Result type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:154
ResultType_< MT1 > RT1
Result type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:153
Header file for the isDefault shim.
Compile time check for Hermitian matrices.This type trait tests whether or not the given template par...
Definition: IsHermitian.h:85
typename TDVecTSMatMultExprTrait< VT, MT >::Type TDVecTSMatMultExprTrait_
Auxiliary alias declaration for the TDVecTSMatMultExprTrait class template.The TDVecTSMatMultExprTrai...
Definition: TDVecTSMatMultExprTrait.h:120
Constraints on the storage order of matrix types.
Generic wrapper for the declherm() function.
Definition: DeclHerm.h:58
Header file for the TDMatDeclSymExprTrait class template.
Header file for the Noop functor.
Header file for the RemoveReference type trait.
typename TDMatDVecMultExprTrait< MT, VT >::Type TDMatDVecMultExprTrait_
Auxiliary alias declaration for the TDMatDVecMultExprTrait class template.The TDMatDVecMultExprTrait_...
Definition: TDMatDVecMultExprTrait.h:120
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:223
typename T::OppositeType OppositeType_
Alias declaration for nested OppositeType type definitions.The OppositeType_ alias declaration provid...
Definition: Aliases.h:243
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:84
Header file for the IsDenseVector type trait.
Generic wrapper for the declupp() function.
Definition: DeclUpp.h:58
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: TDMatTSMatMultExpr.h:375
Evaluation of the expression type of a dense vector/transpose dense matrix multiplication.Via this type trait it is possible to evaluate the resulting expression type of a dense vector/transpose dense matrix multiplication. Given the transpose dense vector type VT and the column-major dense matrix type MT, the nested type Type corresponds to the resulting expression type. In case either VT is not a transpose dense vector type or MT is not a column-major dense matrix type, the resulting data type Type is set to INVALID_TYPE.
Definition: TDVecTDMatMultExprTrait.h:79
Header file for the IsComputation type trait class.
Header file for the IsBuiltin type trait.
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatTSMatMultExpr.h:274
Compile time logical or evaluation.The Or class template performs at compile time a logical or (&#39;&&&#39;)...
Definition: Or.h:101
Header file for the TDMatDVecMultExprTrait class template.
Evaluation of the expression type of a transpose dense matrix/dense vector multiplication.Via this type trait it is possible to evaluate the resulting expression type of a transpose dense matrix/dense vector multiplication. Given the column-major dense matrix type MT and the non-transpose dense vector type VT, the nested type Type corresponds to the resulting expression type. In case either MT is not a column-major dense matrix type or VT is not a non-transpose dense vector type, the resulting data type Type is set to INVALID_TYPE.
Definition: TDMatDVecMultExprTrait.h:79
CompositeType_< MT2 > CT2
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:158
Header file for the IntegralConstant class template.
Compile time evaluation of the number of columns of a matrix.The Columns type trait evaluates the num...
Definition: Columns.h:76
Generic wrapper for the decldiag() function.
Definition: DeclDiag.h:58
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatTSMatMultExpr.h:472
Compile time evaluation of the number of rows of a matrix.The Rows type trait evaluates the number of...
Definition: Rows.h:76
Compile time check for sparse matrix types.This type trait tests whether or not the given template pa...
Definition: IsSparseMatrix.h:78
Header file for the DeclHerm functor.
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TDMatTSMatMultExpr.h:465
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:573
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:403
Header file for the IsUpper type trait.
Header file for the IsColumnVector type trait.
CompositeType_< MT1 > CT1
Composite type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:157
Flag for upper matrices.
Definition: TDMatTSMatMultExpr.h:177
Constraint on the data type.
Generic wrapper for the declsym() function.
Definition: DeclSym.h:58
BLAZE_ALWAYS_INLINE bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition: Matrix.h:677
Header file for the IsResizable type trait.
const DMatDMatMultExpr< T1, T2, false, false, false, false > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:7505
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
DisableIf_< IsDiagonal< MT >, const DMatDeclDiagExpr< MT, SO > > decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given non-diagonal dense matrix expression dm as diagonal.
Definition: DMatDeclDiagExpr.h:841
DisableIf_< IsUpper< MT >, const DMatDeclUppExpr< MT, SO > > declupp(const DenseMatrix< MT, SO > &dm)
Declares the given non-upper dense matrix expression dm as upper.
Definition: DMatDeclUppExpr.h:842
Evaluation of the expression type type of a column operation.Via this type trait it is possible to ev...
Definition: ColumnExprTrait.h:78
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the Bool class template.
Header file for the TDVecTDMatMultExprTrait class template.
Header file for the DeclSym functor.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:61
Header file for the IsExpression type trait class.
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatTSMatMultExpr.h:455
Header file for the TSMatSVecMultExprTrait class template.
Header file for the function trace functionality.
Flag for Hermitian matrices.
Definition: TDMatTSMatMultExpr.h:175