TDMatTSMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
50 #include <blaze/math/Exception.h>
55 #include <blaze/math/Functions.h>
57 #include <blaze/math/shims/Reset.h>
59 #include <blaze/math/SIMD.h>
99 #include <blaze/util/Assert.h>
101 #include <blaze/util/EnableIf.h>
103 #include <blaze/util/InvalidType.h>
105 #include <blaze/util/mpl/And.h>
106 #include <blaze/util/mpl/If.h>
107 #include <blaze/util/mpl/Or.h>
108 #include <blaze/util/Types.h>
110 
111 
112 namespace blaze {
113 
114 //=================================================================================================
115 //
116 // CLASS TDMATTSMATMULTEXPR
117 //
118 //=================================================================================================
119 
120 //*************************************************************************************************
127 template< typename MT1 // Type of the left-hand side dense matrix
128  , typename MT2 > // Type of the right-hand side sparse matrix
129 class TDMatTSMatMultExpr : public DenseMatrix< TDMatTSMatMultExpr<MT1,MT2>, true >
130  , private MatMatMultExpr
131  , private Computation
132 {
133  private:
134  //**Type definitions****************************************************************************
141  //**********************************************************************************************
142 
143  //**********************************************************************************************
145  enum : bool { evaluateLeft = IsComputation<MT1>::value || RequiresEvaluation<MT1>::value };
146  //**********************************************************************************************
147 
148  //**********************************************************************************************
150  enum : bool { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
151  //**********************************************************************************************
152 
153  //**********************************************************************************************
155 
159  template< typename T1, typename T2, typename T3 >
160  struct IsEvaluationRequired {
161  enum : bool { value = ( evaluateLeft || evaluateRight ) };
162  };
164  //**********************************************************************************************
165 
166  //**********************************************************************************************
168 
171  template< typename T1, typename T2, typename T3 >
172  struct UseVectorizedKernel {
173  enum : bool { value = useOptimizedKernels &&
174  !IsDiagonal<T2>::value &&
175  T1::simdEnabled && T2::simdEnabled &&
176  IsColumnMajorMatrix<T1>::value &&
177  AreSIMDCombinable< ElementType_<T1>
178  , ElementType_<T2>
179  , ElementType_<T3> >::value &&
180  HasSIMDAdd< ElementType_<T2>, ElementType_<T3> >::value &&
181  HasSIMDMult< ElementType_<T2>, ElementType_<T3> >::value };
182  };
184  //**********************************************************************************************
185 
186  //**********************************************************************************************
188 
192  template< typename T1, typename T2, typename T3 >
193  struct UseOptimizedKernel {
194  enum : bool { value = useOptimizedKernels &&
195  !UseVectorizedKernel<T1,T2,T3>::value &&
196  !IsDiagonal<T2>::value &&
197  !IsResizable< ElementType_<T1> >::value &&
198  !IsResizable<ET2>::value };
199  };
201  //**********************************************************************************************
202 
203  //**********************************************************************************************
205 
208  template< typename T1, typename T2, typename T3 >
209  struct UseDefaultKernel {
210  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
211  !UseOptimizedKernel<T1,T2,T3>::value };
212  };
214  //**********************************************************************************************
215 
216  public:
217  //**Type definitions****************************************************************************
224  typedef const ElementType ReturnType;
225  typedef const ResultType CompositeType;
226 
228  typedef If_< IsExpression<MT1>, const MT1, const MT1& > LeftOperand;
229 
231  typedef If_< IsExpression<MT2>, const MT2, const MT2& > RightOperand;
232 
235 
238  //**********************************************************************************************
239 
240  //**Compilation flags***************************************************************************
242  enum : bool { simdEnabled = !IsDiagonal<MT1>::value &&
243  MT1::simdEnabled &&
246 
248  enum : bool { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
249  !evaluateRight && MT2::smpAssignable };
250  //**********************************************************************************************
251 
252  //**SIMD properties*****************************************************************************
254  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
255  //**********************************************************************************************
256 
257  //**Constructor*********************************************************************************
263  explicit inline TDMatTSMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
264  : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
265  , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
266  {
267  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
268  }
269  //**********************************************************************************************
270 
271  //**Access operator*****************************************************************************
278  inline ReturnType operator()( size_t i, size_t j ) const {
279  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
280  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
281 
282  if( IsDiagonal<MT1>::value ) {
283  return lhs_(i,i) * rhs_(i,j);
284  }
285  else if( IsDiagonal<MT2>::value ) {
286  return lhs_(i,j) * rhs_(j,j);
287  }
289  const size_t begin( ( IsUpper<MT1>::value )
290  ?( ( IsLower<MT2>::value )
291  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
292  , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
293  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
294  :( ( IsLower<MT2>::value )
295  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
296  :( 0UL ) ) );
297  const size_t end( ( IsLower<MT1>::value )
298  ?( ( IsUpper<MT2>::value )
299  ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
300  , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
301  :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
302  :( ( IsUpper<MT2>::value )
303  ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
304  :( lhs_.columns() ) ) );
305 
306  if( begin >= end ) return ElementType();
307 
308  const size_t n( end - begin );
309 
310  return subvector( row( lhs_, i ), begin, n ) * subvector( column( rhs_, j ), begin, n );
311  }
312  else {
313  return row( lhs_, i ) * column( rhs_, j );
314  }
315  }
316  //**********************************************************************************************
317 
318  //**At function*********************************************************************************
326  inline ReturnType at( size_t i, size_t j ) const {
327  if( i >= lhs_.rows() ) {
328  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
329  }
330  if( j >= rhs_.columns() ) {
331  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
332  }
333  return (*this)(i,j);
334  }
335  //**********************************************************************************************
336 
337  //**Rows function*******************************************************************************
342  inline size_t rows() const noexcept {
343  return lhs_.rows();
344  }
345  //**********************************************************************************************
346 
347  //**Columns function****************************************************************************
352  inline size_t columns() const noexcept {
353  return rhs_.columns();
354  }
355  //**********************************************************************************************
356 
357  //**Left operand access*************************************************************************
362  inline LeftOperand leftOperand() const noexcept {
363  return lhs_;
364  }
365  //**********************************************************************************************
366 
367  //**Right operand access************************************************************************
372  inline RightOperand rightOperand() const noexcept {
373  return rhs_;
374  }
375  //**********************************************************************************************
376 
377  //**********************************************************************************************
383  template< typename T >
384  inline bool canAlias( const T* alias ) const noexcept {
385  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
386  }
387  //**********************************************************************************************
388 
389  //**********************************************************************************************
395  template< typename T >
396  inline bool isAliased( const T* alias ) const noexcept {
397  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
398  }
399  //**********************************************************************************************
400 
401  //**********************************************************************************************
406  inline bool isAligned() const noexcept {
407  return lhs_.isAligned();
408  }
409  //**********************************************************************************************
410 
411  //**********************************************************************************************
416  inline bool canSMPAssign() const noexcept {
417  return ( rows() * columns() >= SMP_TDMATTSMATMULT_THRESHOLD );
418  }
419  //**********************************************************************************************
420 
421  private:
422  //**Member variables****************************************************************************
423  LeftOperand lhs_;
424  RightOperand rhs_;
425  //**********************************************************************************************
426 
427  //**Assignment to dense matrices****************************************************************
440  template< typename MT // Type of the target dense matrix
441  , bool SO > // Storage order of the target dense matrix
442  friend inline void assign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
443  {
445 
446  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
447  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
448 
449  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
450  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
451 
452  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
453  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
454  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
455  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
456  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
457  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
458 
459  TDMatTSMatMultExpr::selectAssignKernel( ~lhs, A, B );
460  }
462  //**********************************************************************************************
463 
464  //**Default assignment to dense matrices********************************************************
478  template< typename MT3 // Type of the left-hand side target matrix
479  , typename MT4 // Type of the left-hand side matrix operand
480  , typename MT5 > // Type of the right-hand side matrix operand
482  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
483  {
485 
486  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
487 
488  reset( C );
489 
490  for( size_t ii=0UL; ii<A.rows(); ii+=block )
491  {
492  const size_t itmp( min( ii+block, A.rows() ) );
493 
494  for( size_t j=0UL; j<B.columns(); ++j )
495  {
496  ConstIterator element( B.begin(j) );
497  const ConstIterator end( B.end(j) );
498 
499  for( ; element!=end; ++element )
500  {
501  const size_t j1( element->index() );
502 
504  {
505  C(j1,j) = A(j1,j1) * element->value();
506  }
507  else
508  {
509  const size_t ibegin( ( IsLower<MT4>::value )
510  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
511  :( ii ) );
512  const size_t iend( ( IsUpper<MT4>::value )
513  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
514  :( itmp ) );
515 
516  if( IsTriangular<MT4>::value && ibegin >= iend )
517  continue;
518 
519  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
520 
521  for( size_t i=ibegin; i<iend; ++i ) {
522  if( isDefault( C(i,j) ) )
523  C(i,j) = A(i,j1) * element->value();
524  else
525  C(i,j) += A(i,j1) * element->value();
526  }
527  }
528  }
529  }
530  }
531  }
533  //**********************************************************************************************
534 
535  //**Optimized assignment to dense matrices******************************************************
549  template< typename MT3 // Type of the left-hand side target matrix
550  , typename MT4 // Type of the left-hand side matrix operand
551  , typename MT5 > // Type of the right-hand side matrix operand
552  static inline EnableIf_< UseOptimizedKernel<MT3,MT4,MT5> >
553  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
554  {
555  typedef ConstIterator_<MT5> ConstIterator;
556 
557  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
558 
559  reset( C );
560 
561  for( size_t ii=0UL; ii<A.rows(); ii+=block )
562  {
563  const size_t itmp( min( ii+block, A.rows() ) );
564 
565  for( size_t j=0UL; j<B.columns(); ++j )
566  {
567  const ConstIterator end( B.end(j) );
568  ConstIterator element( B.begin(j) );
569 
570  const size_t nonzeros( B.nonZeros(j) );
571  const size_t kpos( nonzeros & size_t(-4) );
572  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
573 
574  for( size_t k=0UL; k<kpos; k+=4UL )
575  {
576  const size_t j1( element->index() );
577  const ET2 v1( element->value() );
578  ++element;
579  const size_t j2( element->index() );
580  const ET2 v2( element->value() );
581  ++element;
582  const size_t j3( element->index() );
583  const ET2 v3( element->value() );
584  ++element;
585  const size_t j4( element->index() );
586  const ET2 v4( element->value() );
587  ++element;
588 
589  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
590 
591  const size_t ibegin( ( IsLower<MT4>::value )
592  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
593  :( ii ) );
594  const size_t iend( ( IsUpper<MT4>::value )
595  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
596  :( itmp ) );
597 
598  if( IsTriangular<MT4>::value && ibegin >= iend )
599  continue;
600 
601  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
602 
603  const size_t inum( iend - ibegin );
604  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
605  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
606 
607  for( size_t i=ibegin; i<ipos; i+=4UL ) {
608  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
609  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
610  C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
611  C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
612  }
613  for( size_t i=ipos; i<iend; ++i ) {
614  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
615  }
616  }
617 
618  for( ; element!=end; ++element )
619  {
620  const size_t j1( element->index() );
621  const ET2 v1( element->value() );
622 
623  const size_t ibegin( ( IsLower<MT4>::value )
624  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
625  :( ii ) );
626  const size_t iend( ( IsUpper<MT4>::value )
627  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
628  :( itmp ) );
629 
630  if( IsTriangular<MT4>::value && ibegin >= iend )
631  continue;
632 
633  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
634 
635  const size_t inum( iend - ibegin );
636  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
637  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
638 
639  for( size_t i=ibegin; i<ipos; i+=4UL ) {
640  C(i ,j) += A(i ,j1) * v1;
641  C(i+1UL,j) += A(i+1UL,j1) * v1;
642  C(i+2UL,j) += A(i+2UL,j1) * v1;
643  C(i+3UL,j) += A(i+3UL,j1) * v1;
644  }
645  for( size_t i=ipos; i<iend; ++i ) {
646  C(i,j) += A(i,j1) * v1;
647  }
648  }
649  }
650  }
651  }
653  //**********************************************************************************************
654 
655  //**Vectorized assignment to column-major dense matrices****************************************
669  template< typename MT3 // Type of the left-hand side target matrix
670  , typename MT4 // Type of the left-hand side matrix operand
671  , typename MT5 > // Type of the right-hand side matrix operand
672  static inline EnableIf_< UseVectorizedKernel<MT3,MT4,MT5> >
673  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
674  {
675  typedef ConstIterator_<MT5> ConstIterator;
676 
677  const bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
678 
679  reset( C );
680 
681  for( size_t j=0UL; j<B.columns(); ++j )
682  {
683  const ConstIterator end( B.end(j) );
684  ConstIterator element( B.begin(j) );
685 
686  const size_t nonzeros( B.nonZeros(j) );
687  const size_t kpos( nonzeros & size_t(-4) );
688  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
689 
690  for( size_t k=0UL; k<kpos; k+=4UL )
691  {
692  const size_t j1( element->index() );
693  const ET2 v1( element->value() );
694  ++element;
695  const size_t j2( element->index() );
696  const ET2 v2( element->value() );
697  ++element;
698  const size_t j3( element->index() );
699  const ET2 v3( element->value() );
700  ++element;
701  const size_t j4( element->index() );
702  const ET2 v4( element->value() );
703  ++element;
704 
705  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
706 
707  const SIMDType xmm1( set( v1 ) );
708  const SIMDType xmm2( set( v2 ) );
709  const SIMDType xmm3( set( v3 ) );
710  const SIMDType xmm4( set( v4 ) );
711 
712  const size_t ibegin( ( IsLower<MT4>::value )
713  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
714  :( 0UL ) );
715  const size_t iend( ( IsUpper<MT4>::value )
716  ?( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL )
717  :( A.rows() ) );
718  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
719 
720  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
721  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
722 
723  size_t i( ibegin );
724 
725  for( ; i<ipos; i+=SIMDSIZE ) {
726  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
727  }
728  for( ; remainder && i<iend; ++i ) {
729  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
730  }
731  }
732 
733  for( ; element!=end; ++element )
734  {
735  const size_t j1( element->index() );
736  const ET2 v1( element->value() );
737 
738  const SIMDType xmm1( set( v1 ) );
739 
740  const size_t ibegin( ( IsLower<MT4>::value )
741  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
742  :( 0UL ) );
743  const size_t iend( ( IsUpper<MT4>::value )
744  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
745  :( A.rows() ) );
746  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
747 
748  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
749  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
750 
751  size_t i( ibegin );
752 
753  for( ; i<ipos; i+=SIMDSIZE ) {
754  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 );
755  }
756  for( ; remainder && i<iend; ++i ) {
757  C(i,j) += A(i,j1) * v1;
758  }
759  }
760  }
761  }
763  //**********************************************************************************************
764 
765  //**Assignment to sparse matrices***************************************************************
778  template< typename MT // Type of the target sparse matrix
779  , bool SO > // Storage order of the target sparse matrix
780  friend inline void assign( SparseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
781  {
783 
784  typedef IfTrue_< SO, ResultType, OppositeType > TmpType;
785 
791  BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE( CompositeType_<TmpType> );
792 
793  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
794  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
795 
796  const TmpType tmp( serial( rhs ) );
797  assign( ~lhs, tmp );
798  }
800  //**********************************************************************************************
801 
802  //**Addition assignment to dense matrices*******************************************************
815  template< typename MT // Type of the target dense matrix
816  , bool SO > // Storage order of the target dense matrix
817  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
818  {
820 
821  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
822  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
823 
824  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
825  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
826 
827  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
828  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
829  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
830  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
831  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
832  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
833 
834  TDMatTSMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
835  }
837  //**********************************************************************************************
838 
839  //**Default addition assignment to dense matrices***********************************************
853  template< typename MT3 // Type of the left-hand side target matrix
854  , typename MT4 // Type of the left-hand side matrix operand
855  , typename MT5 > // Type of the right-hand side matrix operand
856  static inline EnableIf_< UseDefaultKernel<MT3,MT4,MT5> >
857  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
858  {
859  typedef ConstIterator_<MT5> ConstIterator;
860 
861  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
862 
863  for( size_t ii=0UL; ii<A.rows(); ii+=block )
864  {
865  const size_t itmp( min( ii+block, A.rows() ) );
866 
867  for( size_t j=0UL; j<B.columns(); ++j )
868  {
869  ConstIterator element( B.begin(j) );
870  const ConstIterator end( B.end(j) );
871 
872  for( ; element!=end; ++element )
873  {
874  const size_t j1( element->index() );
875 
876  if( IsDiagonal<MT4>::value )
877  {
878  C(j1,j) += A(j1,j1) * element->value();
879  }
880  else
881  {
882  const size_t ibegin( ( IsLower<MT4>::value )
883  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
884  :( ii ) );
885  const size_t iend( ( IsUpper<MT4>::value )
886  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
887  :( itmp ) );
888 
889  if( IsTriangular<MT4>::value && ibegin >= iend )
890  continue;
891 
892  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
893 
894  const size_t inum( iend - ibegin );
895  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
896  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
897 
898  for( size_t i=ibegin; i<ipos; i+=4UL ) {
899  C(i ,j) += A(i ,j1) * element->value();
900  C(i+1UL,j) += A(i+1UL,j1) * element->value();
901  C(i+2UL,j) += A(i+2UL,j1) * element->value();
902  C(i+3UL,j) += A(i+3UL,j1) * element->value();
903  }
904  for( size_t i=ipos; i<iend; ++i ) {
905  C(i,j) += A(i,j1) * element->value();
906  }
907  }
908  }
909  }
910  }
911  }
913  //**********************************************************************************************
914 
915  //**Optimized addition assignment to dense matrices*********************************************
929  template< typename MT3 // Type of the left-hand side target matrix
930  , typename MT4 // Type of the left-hand side matrix operand
931  , typename MT5 > // Type of the right-hand side matrix operand
932  static inline EnableIf_< UseOptimizedKernel<MT3,MT4,MT5> >
933  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
934  {
935  typedef ConstIterator_<MT5> ConstIterator;
936 
937  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
938 
939  for( size_t ii=0UL; ii<A.rows(); ii+=block )
940  {
941  const size_t itmp( min( ii+block, A.rows() ) );
942 
943  for( size_t j=0UL; j<B.columns(); ++j )
944  {
945  const ConstIterator end( B.end(j) );
946  ConstIterator element( B.begin(j) );
947 
948  const size_t nonzeros( B.nonZeros(j) );
949  const size_t kpos( nonzeros & size_t(-4) );
950  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
951 
952  for( size_t k=0UL; k<kpos; k+=4UL )
953  {
954  const size_t j1( element->index() );
955  const ET2 v1( element->value() );
956  ++element;
957  const size_t j2( element->index() );
958  const ET2 v2( element->value() );
959  ++element;
960  const size_t j3( element->index() );
961  const ET2 v3( element->value() );
962  ++element;
963  const size_t j4( element->index() );
964  const ET2 v4( element->value() );
965  ++element;
966 
967  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
968 
969  const size_t ibegin( ( IsLower<MT4>::value )
970  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
971  :( ii ) );
972  const size_t iend( ( IsUpper<MT4>::value )
973  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
974  :( itmp ) );
975 
976  if( IsTriangular<MT4>::value && ibegin >= iend )
977  continue;
978 
979  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
980 
981  const size_t inum( iend - ibegin );
982  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
983  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
984 
985  for( size_t i=ibegin; i<ipos; i+=4UL ) {
986  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
987  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
988  C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
989  C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
990  }
991  for( size_t i=ipos; i<iend; ++i ) {
992  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
993  }
994  }
995 
996  for( ; element!=end; ++element )
997  {
998  const size_t j1( element->index() );
999  const ET2 v1( element->value() );
1000 
1001  const size_t ibegin( ( IsLower<MT4>::value )
1002  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1003  :( ii ) );
1004  const size_t iend( ( IsUpper<MT4>::value )
1005  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1006  :( itmp ) );
1007 
1008  if( IsTriangular<MT4>::value && ibegin >= iend )
1009  continue;
1010 
1011  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1012 
1013  const size_t inum( iend - ibegin );
1014  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1015  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1016 
1017  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1018  C(i ,j) += A(i ,j1) * v1;
1019  C(i+1UL,j) += A(i+1UL,j1) * v1;
1020  C(i+2UL,j) += A(i+2UL,j1) * v1;
1021  C(i+3UL,j) += A(i+3UL,j1) * v1;
1022  }
1023  for( size_t i=ipos; i<iend; ++i ) {
1024  C(i,j) += A(i,j1) * v1;
1025  }
1026  }
1027  }
1028  }
1029  }
1031  //**********************************************************************************************
1032 
1033  //**Vectorized addition assignment to column-major dense matrices*******************************
1047  template< typename MT3 // Type of the left-hand side target matrix
1048  , typename MT4 // Type of the left-hand side matrix operand
1049  , typename MT5 > // Type of the right-hand side matrix operand
1050  static inline EnableIf_< UseVectorizedKernel<MT3,MT4,MT5> >
1051  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1052  {
1053  typedef ConstIterator_<MT5> ConstIterator;
1054 
1055  const bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
1056 
1057  for( size_t j=0UL; j<B.columns(); ++j )
1058  {
1059  const ConstIterator end( B.end(j) );
1060  ConstIterator element( B.begin(j) );
1061 
1062  const size_t nonzeros( B.nonZeros(j) );
1063  const size_t kpos( nonzeros & size_t(-4) );
1064  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1065 
1066  for( size_t k=0UL; k<kpos; k+=4UL )
1067  {
1068  const size_t j1( element->index() );
1069  const ET2 v1( element->value() );
1070  ++element;
1071  const size_t j2( element->index() );
1072  const ET2 v2( element->value() );
1073  ++element;
1074  const size_t j3( element->index() );
1075  const ET2 v3( element->value() );
1076  ++element;
1077  const size_t j4( element->index() );
1078  const ET2 v4( element->value() );
1079  ++element;
1080 
1081  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1082 
1083  const SIMDType xmm1( set( v1 ) );
1084  const SIMDType xmm2( set( v2 ) );
1085  const SIMDType xmm3( set( v3 ) );
1086  const SIMDType xmm4( set( v4 ) );
1087 
1088  const size_t ibegin( ( IsLower<MT4>::value )
1089  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1090  :( 0UL ) );
1091  const size_t iend( ( IsUpper<MT4>::value )
1092  ?( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL )
1093  :( A.rows() ) );
1094  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1095 
1096  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1097  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1098 
1099  size_t i( ibegin );
1100 
1101  for( ; i<ipos; i+=SIMDSIZE ) {
1102  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
1103  }
1104  for( ; remainder && i<iend; ++i ) {
1105  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1106  }
1107  }
1108 
1109  for( ; element!=end; ++element )
1110  {
1111  const size_t j1( element->index() );
1112  const ET2 v1( element->value() );
1113 
1114  const SIMDType xmm1( set( v1 ) );
1115 
1116  const size_t ibegin( ( IsLower<MT4>::value )
1117  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1118  :( 0UL ) );
1119  const size_t iend( ( IsUpper<MT4>::value )
1120  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
1121  :( A.rows() ) );
1122  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1123 
1124  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1125  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1126 
1127  size_t i( ibegin );
1128 
1129  for( ; i<ipos; i+=SIMDSIZE ) {
1130  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 );
1131  }
1132  for( ; remainder && i<iend; ++i ) {
1133  C(i,j) += A(i,j1) * v1;
1134  }
1135  }
1136  }
1137  }
1139  //**********************************************************************************************
1140 
1141  //**Addition assignment to sparse matrices******************************************************
1142  // No special implementation for the addition assignment to sparse matrices.
1143  //**********************************************************************************************
1144 
1145  //**Subtraction assignment to dense matrices****************************************************
1158  template< typename MT // Type of the target dense matrix
1159  , bool SO > // Storage order of the target dense matrix
1160  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1161  {
1163 
1164  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1165  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1166 
1167  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1168  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1169 
1170  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1171  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1172  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1173  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1174  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1175  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1176 
1177  TDMatTSMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1178  }
1180  //**********************************************************************************************
1181 
1182  //**Default subtraction assignment to dense matrices********************************************
1196  template< typename MT3 // Type of the left-hand side target matrix
1197  , typename MT4 // Type of the left-hand side matrix operand
1198  , typename MT5 > // Type of the right-hand side matrix operand
1199  static inline EnableIf_< UseDefaultKernel<MT3,MT4,MT5> >
1200  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1201  {
1202  typedef ConstIterator_<MT5> ConstIterator;
1203 
1204  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
1205 
1206  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1207  {
1208  const size_t itmp( min( ii+block, A.rows() ) );
1209 
1210  for( size_t j=0UL; j<B.columns(); ++j )
1211  {
1212  ConstIterator element( B.begin(j) );
1213  const ConstIterator end( B.end(j) );
1214 
1215  for( ; element!=end; ++element )
1216  {
1217  const size_t j1( element->index() );
1218 
1219  if( IsDiagonal<MT4>::value )
1220  {
1221  C(j1,j) -= A(j1,j1) * element->value();
1222  }
1223  else
1224  {
1225  const size_t ibegin( ( IsLower<MT4>::value )
1226  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1227  :( ii ) );
1228  const size_t iend( ( IsUpper<MT4>::value )
1229  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1230  :( itmp ) );
1231 
1232  if( IsTriangular<MT4>::value && ibegin >= iend )
1233  continue;
1234 
1235  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1236 
1237  const size_t inum( iend - ibegin );
1238  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1239  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1240 
1241  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1242  C(i ,j) -= A(i ,j1) * element->value();
1243  C(i+1UL,j) -= A(i+1UL,j1) * element->value();
1244  C(i+2UL,j) -= A(i+2UL,j1) * element->value();
1245  C(i+3UL,j) -= A(i+3UL,j1) * element->value();
1246  }
1247  for( size_t i=ipos; i<iend; ++i ) {
1248  C(i,j) -= A(i,j1) * element->value();
1249  }
1250  }
1251  }
1252  }
1253  }
1254  }
1256  //**********************************************************************************************
1257 
1258  //**Optimized subtraction assignment to dense matrices******************************************
1272  template< typename MT3 // Type of the left-hand side target matrix
1273  , typename MT4 // Type of the left-hand side matrix operand
1274  , typename MT5 > // Type of the right-hand side matrix operand
1275  static inline EnableIf_< UseOptimizedKernel<MT3,MT4,MT5> >
1276  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1277  {
1278  typedef ConstIterator_<MT5> ConstIterator;
1279 
1280  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
1281 
1282  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1283  {
1284  const size_t itmp( min( ii+block, A.rows() ) );
1285 
1286  for( size_t j=0UL; j<B.columns(); ++j )
1287  {
1288  const ConstIterator end( B.end(j) );
1289  ConstIterator element( B.begin(j) );
1290 
1291  const size_t nonzeros( B.nonZeros(j) );
1292  const size_t kpos( nonzeros & size_t(-4) );
1293  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1294 
1295  for( size_t k=0UL; k<kpos; k+=4UL )
1296  {
1297  const size_t j1( element->index() );
1298  const ET2 v1( element->value() );
1299  ++element;
1300  const size_t j2( element->index() );
1301  const ET2 v2( element->value() );
1302  ++element;
1303  const size_t j3( element->index() );
1304  const ET2 v3( element->value() );
1305  ++element;
1306  const size_t j4( element->index() );
1307  const ET2 v4( element->value() );
1308  ++element;
1309 
1310  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1311 
1312  const size_t ibegin( ( IsLower<MT4>::value )
1313  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1314  :( ii ) );
1315  const size_t iend( ( IsUpper<MT4>::value )
1316  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
1317  :( itmp ) );
1318 
1319  if( IsTriangular<MT4>::value && ibegin >= iend )
1320  continue;
1321 
1322  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1323 
1324  const size_t inum( iend - ibegin );
1325  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1326  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1327 
1328  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1329  C(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1330  C(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1331  C(i+2UL,j) -= A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1332  C(i+3UL,j) -= A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1333  }
1334  for( size_t i=ipos; i<iend; ++i ) {
1335  C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1336  }
1337  }
1338 
1339  for( ; element!=end; ++element )
1340  {
1341  const size_t j1( element->index() );
1342  const ET2 v1( element->value() );
1343 
1344  const size_t ibegin( ( IsLower<MT4>::value )
1345  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1346  :( ii ) );
1347  const size_t iend( ( IsUpper<MT4>::value )
1348  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1349  :( itmp ) );
1350 
1351  if( IsTriangular<MT4>::value && ibegin >= iend )
1352  continue;
1353 
1354  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1355 
1356  const size_t inum( iend - ibegin );
1357  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1358  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1359 
1360  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1361  C(i ,j) -= A(i ,j1) * v1;
1362  C(i+1UL,j) -= A(i+1UL,j1) * v1;
1363  C(i+2UL,j) -= A(i+2UL,j1) * v1;
1364  C(i+3UL,j) -= A(i+3UL,j1) * v1;
1365  }
1366  for( size_t i=ipos; i<iend; ++i ) {
1367  C(i,j) -= A(i,j1) * v1;
1368  }
1369  }
1370  }
1371  }
1372  }
1374  //**********************************************************************************************
1375 
1376  //**Vectorized subtraction assignment to column-major dense matrices****************************
1390  template< typename MT3 // Type of the left-hand side target matrix
1391  , typename MT4 // Type of the left-hand side matrix operand
1392  , typename MT5 > // Type of the right-hand side matrix operand
1393  static inline EnableIf_< UseVectorizedKernel<MT3,MT4,MT5> >
1394  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1395  {
1396  typedef ConstIterator_<MT5> ConstIterator;
1397 
1398  const bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
1399 
1400  for( size_t j=0UL; j<B.columns(); ++j )
1401  {
1402  const ConstIterator end( B.end(j) );
1403  ConstIterator element( B.begin(j) );
1404 
1405  const size_t nonzeros( B.nonZeros(j) );
1406  const size_t kpos( nonzeros & size_t(-4) );
1407  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1408 
1409  for( size_t k=0UL; k<kpos; k+=4UL )
1410  {
1411  const size_t j1( element->index() );
1412  const ET2 v1( element->value() );
1413  ++element;
1414  const size_t j2( element->index() );
1415  const ET2 v2( element->value() );
1416  ++element;
1417  const size_t j3( element->index() );
1418  const ET2 v3( element->value() );
1419  ++element;
1420  const size_t j4( element->index() );
1421  const ET2 v4( element->value() );
1422  ++element;
1423 
1424  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1425 
1426  const SIMDType xmm1( set( v1 ) );
1427  const SIMDType xmm2( set( v2 ) );
1428  const SIMDType xmm3( set( v3 ) );
1429  const SIMDType xmm4( set( v4 ) );
1430 
1431  const size_t ibegin( ( IsLower<MT4>::value )
1432  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1433  :( 0UL ) );
1434  const size_t iend( ( IsUpper<MT4>::value )
1435  ?( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL )
1436  :( A.rows() ) );
1437  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1438 
1439  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1440  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1441 
1442  size_t i( ibegin );
1443 
1444  for( ; i<ipos; i+=SIMDSIZE ) {
1445  C.store( i, j, C.load(i,j) - A.load(i,j1) * xmm1 - A.load(i,j2) * xmm2 - A.load(i,j3) * xmm3 - A.load(i,j4) * xmm4 );
1446  }
1447  for( ; remainder && i<iend; ++i ) {
1448  C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1449  }
1450  }
1451 
1452  for( ; element!=end; ++element )
1453  {
1454  const size_t j1( element->index() );
1455  const ET2 v1( element->value() );
1456 
1457  const SIMDType xmm1( set( v1 ) );
1458 
1459  const size_t ibegin( ( IsLower<MT4>::value )
1460  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1461  :( 0UL ) );
1462  const size_t iend( ( IsUpper<MT4>::value )
1463  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
1464  :( A.rows() ) );
1465  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1466 
1467  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1468  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
1469 
1470  size_t i( ibegin );
1471 
1472  for( ; i<ipos; i+=SIMDSIZE ) {
1473  C.store( i, j, C.load(i,j) - A.load(i,j1) * xmm1 );
1474  }
1475  for( ; remainder && i<iend; ++i ) {
1476  C(i,j) -= A(i,j1) * v1;
1477  }
1478  }
1479  }
1480  }
1482  //**********************************************************************************************
1483 
1484  //**Subtraction assignment to sparse matrices***************************************************
1485  // No special implementation for the subtraction assignment to sparse matrices.
1486  //**********************************************************************************************
1487 
1488  //**Multiplication assignment to dense matrices*************************************************
1489  // No special implementation for the multiplication assignment to dense matrices.
1490  //**********************************************************************************************
1491 
1492  //**Multiplication assignment to sparse matrices************************************************
1493  // No special implementation for the multiplication assignment to sparse matrices.
1494  //**********************************************************************************************
1495 
1496  //**SMP assignment to dense matrices************************************************************
1511  template< typename MT // Type of the target dense matrix
1512  , bool SO > // Storage order of the target dense matrix
1513  friend inline EnableIf_< IsEvaluationRequired<MT,MT1,MT2> >
1514  smpAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1515  {
1517 
1518  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1519  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1520 
1521  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1522  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1523 
1524  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1525  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1526  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1527  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1528  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1529  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1530 
1531  smpAssign( ~lhs, A * B );
1532  }
1534  //**********************************************************************************************
1535 
1536  //**SMP assignment to sparse matrices***********************************************************
1551  template< typename MT // Type of the target sparse matrix
1552  , bool SO > // Storage order of the target sparse matrix
1553  friend inline EnableIf_< IsEvaluationRequired<MT,MT1,MT2> >
1554  smpAssign( SparseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1555  {
1557 
1558  typedef IfTrue_< SO, ResultType, OppositeType > TmpType;
1559 
1565  BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE( CompositeType_<TmpType> );
1566 
1567  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1568  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1569 
1570  const TmpType tmp( rhs );
1571  smpAssign( ~lhs, tmp );
1572  }
1574  //**********************************************************************************************
1575 
1576  //**SMP addition assignment to dense matrices***************************************************
1591  template< typename MT // Type of the target dense matrix
1592  , bool SO > // Storage order of the target dense matrix
1593  friend inline EnableIf_< IsEvaluationRequired<MT,MT1,MT2> >
1594  smpAddAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1595  {
1597 
1598  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1599  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1600 
1601  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1602  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1603 
1604  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1605  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1606  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1607  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1608  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1609  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1610 
1611  smpAddAssign( ~lhs, A * B );
1612  }
1614  //**********************************************************************************************
1615 
1616  //**SMP addition assignment to sparse matrices**************************************************
1617  // No special implementation for the SMP addition assignment to sparse matrices.
1618  //**********************************************************************************************
1619 
1620  //**SMP subtraction assignment to dense matrices************************************************
1635  template< typename MT // Type of the target dense matrix
1636  , bool SO > // Storage order of the target dense matrix
1637  friend inline EnableIf_< IsEvaluationRequired<MT,MT1,MT2> >
1638  smpSubAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1639  {
1641 
1642  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1643  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1644 
1645  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1646  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1647 
1648  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1649  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1650  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1651  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1652  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1653  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1654 
1655  smpSubAssign( ~lhs, A * B );
1656  }
1658  //**********************************************************************************************
1659 
1660  //**SMP subtraction assignment to sparse matrices***********************************************
1661  // No special implementation for the SMP subtraction assignment to sparse matrices.
1662  //**********************************************************************************************
1663 
1664  //**SMP multiplication assignment to dense matrices*********************************************
1665  // No special implementation for the SMP multiplication assignment to dense matrices.
1666  //**********************************************************************************************
1667 
1668  //**SMP multiplication assignment to sparse matrices********************************************
1669  // No special implementation for the SMP multiplication assignment to sparse matrices.
1670  //**********************************************************************************************
1671 
1672  //**Compile time checks*************************************************************************
1680  //**********************************************************************************************
1681 };
1682 //*************************************************************************************************
1683 
1684 
1685 
1686 
1687 //=================================================================================================
1688 //
1689 // GLOBAL BINARY ARITHMETIC OPERATORS
1690 //
1691 //=================================================================================================
1692 
1693 //*************************************************************************************************
1722 template< typename T1 // Type of the left-hand side dense matrix
1723  , typename T2 > // Type of the right-hand side sparse matrix
1724 inline const TDMatTSMatMultExpr<T1,T2>
1726 {
1728 
1729  if( (~lhs).columns() != (~rhs).rows() ) {
1730  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1731  }
1732 
1733  return TDMatTSMatMultExpr<T1,T2>( ~lhs, ~rhs );
1734 }
1735 //*************************************************************************************************
1736 
1737 
1738 
1739 
1740 //=================================================================================================
1741 //
1742 // ROWS SPECIALIZATIONS
1743 //
1744 //=================================================================================================
1745 
1746 //*************************************************************************************************
1748 template< typename MT1, typename MT2 >
1749 struct Rows< TDMatTSMatMultExpr<MT1,MT2> > : public Rows<MT1>
1750 {};
1752 //*************************************************************************************************
1753 
1754 
1755 
1756 
1757 //=================================================================================================
1758 //
1759 // COLUMNS SPECIALIZATIONS
1760 //
1761 //=================================================================================================
1762 
1763 //*************************************************************************************************
1765 template< typename MT1, typename MT2 >
1766 struct Columns< TDMatTSMatMultExpr<MT1,MT2> > : public Columns<MT2>
1767 {};
1769 //*************************************************************************************************
1770 
1771 
1772 
1773 
1774 //=================================================================================================
1775 //
1776 // ISALIGNED SPECIALIZATIONS
1777 //
1778 //=================================================================================================
1779 
1780 //*************************************************************************************************
1782 template< typename MT1, typename MT2 >
1783 struct IsAligned< TDMatTSMatMultExpr<MT1,MT2> >
1784  : public BoolConstant< IsAligned<MT1>::value >
1785 {};
1787 //*************************************************************************************************
1788 
1789 
1790 
1791 
1792 //=================================================================================================
1793 //
1794 // ISLOWER SPECIALIZATIONS
1795 //
1796 //=================================================================================================
1797 
1798 //*************************************************************************************************
1800 template< typename MT1, typename MT2 >
1801 struct IsLower< TDMatTSMatMultExpr<MT1,MT2> >
1802  : public BoolConstant< And< IsLower<MT1>, IsLower<MT2> >::value >
1803 {};
1805 //*************************************************************************************************
1806 
1807 
1808 
1809 
1810 //=================================================================================================
1811 //
1812 // ISUNILOWER SPECIALIZATIONS
1813 //
1814 //=================================================================================================
1815 
1816 //*************************************************************************************************
1818 template< typename MT1, typename MT2 >
1819 struct IsUniLower< TDMatTSMatMultExpr<MT1,MT2> >
1820  : public BoolConstant< And< IsUniLower<MT1>, IsUniLower<MT2> >::value >
1821 {};
1823 //*************************************************************************************************
1824 
1825 
1826 
1827 
1828 //=================================================================================================
1829 //
1830 // ISSTRICTLYLOWER SPECIALIZATIONS
1831 //
1832 //=================================================================================================
1833 
1834 //*************************************************************************************************
1836 template< typename MT1, typename MT2 >
1837 struct IsStrictlyLower< TDMatTSMatMultExpr<MT1,MT2> >
1838  : public BoolConstant< Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
1839  , And< IsStrictlyLower<MT2>, IsLower<MT1> > >::value >
1840 {};
1842 //*************************************************************************************************
1843 
1844 
1845 
1846 
1847 //=================================================================================================
1848 //
1849 // ISUPPER SPECIALIZATIONS
1850 //
1851 //=================================================================================================
1852 
1853 //*************************************************************************************************
1855 template< typename MT1, typename MT2 >
1856 struct IsUpper< TDMatTSMatMultExpr<MT1,MT2> >
1857  : public BoolConstant< And< IsUpper<MT1>, IsUpper<MT2> >::value >
1858 {};
1860 //*************************************************************************************************
1861 
1862 
1863 
1864 
1865 //=================================================================================================
1866 //
1867 // ISUNIUPPER SPECIALIZATIONS
1868 //
1869 //=================================================================================================
1870 
1871 //*************************************************************************************************
1873 template< typename MT1, typename MT2 >
1874 struct IsUniUpper< TDMatTSMatMultExpr<MT1,MT2> >
1875  : public BoolConstant< And< IsUniUpper<MT1>, IsUniUpper<MT2> >::value >
1876 {};
1878 //*************************************************************************************************
1879 
1880 
1881 
1882 
1883 //=================================================================================================
1884 //
1885 // ISSTRICTLYUPPER SPECIALIZATIONS
1886 //
1887 //=================================================================================================
1888 
1889 //*************************************************************************************************
1891 template< typename MT1, typename MT2 >
1892 struct IsStrictlyUpper< TDMatTSMatMultExpr<MT1,MT2> >
1893  : public BoolConstant< Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
1894  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > >::value >
1895 {};
1897 //*************************************************************************************************
1898 
1899 
1900 
1901 
1902 //=================================================================================================
1903 //
1904 // EXPRESSION TRAIT SPECIALIZATIONS
1905 //
1906 //=================================================================================================
1907 
1908 //*************************************************************************************************
1910 template< typename MT1, typename MT2, typename VT >
1911 struct TDMatDVecMultExprTrait< TDMatTSMatMultExpr<MT1,MT2>, VT >
1912 {
1913  public:
1914  //**********************************************************************************************
1915  using Type = If_< And< IsDenseMatrix<MT1>, IsColumnMajorMatrix<MT1>
1916  , IsSparseMatrix<MT2>, IsColumnMajorMatrix<MT2>
1917  , IsDenseVector<VT>, IsColumnVector<VT> >
1918  , TDMatDVecMultExprTrait_< MT1, TSMatDVecMultExprTrait_<MT2,VT> >
1919  , INVALID_TYPE >;
1920  //**********************************************************************************************
1921 };
1923 //*************************************************************************************************
1924 
1925 
1926 //*************************************************************************************************
1928 template< typename MT1, typename MT2, typename VT >
1929 struct TDMatSVecMultExprTrait< TDMatTSMatMultExpr<MT1,MT2>, VT >
1930 {
1931  public:
1932  //**********************************************************************************************
1933  using Type = If_< And< IsDenseMatrix<MT1>, IsColumnMajorMatrix<MT1>
1934  , IsSparseMatrix<MT2>, IsColumnMajorMatrix<MT2>
1935  , IsSparseVector<VT>, IsColumnVector<VT> >
1936  , TDMatDVecMultExprTrait_< MT1, TSMatDVecMultExprTrait_<MT2,VT> >
1937  , INVALID_TYPE >;
1938  //**********************************************************************************************
1939 };
1941 //*************************************************************************************************
1942 
1943 
1944 //*************************************************************************************************
1946 template< typename VT, typename MT1, typename MT2 >
1947 struct TDVecTDMatMultExprTrait< VT, TDMatTSMatMultExpr<MT1,MT2> >
1948 {
1949  public:
1950  //**********************************************************************************************
1951  using Type = If_< And< IsDenseVector<VT>, IsRowVector<VT>
1952  , IsDenseMatrix<MT1>, IsColumnMajorMatrix<MT1>
1953  , IsSparseMatrix<MT2>, IsColumnMajorMatrix<MT2> >
1954  , TDVecTSMatMultExprTrait_< TDVecTDMatMultExprTrait_<VT,MT1>, MT2 >
1955  , INVALID_TYPE >;
1956  //**********************************************************************************************
1957 };
1959 //*************************************************************************************************
1960 
1961 
1962 //*************************************************************************************************
1964 template< typename VT, typename MT1, typename MT2 >
1965 struct TSVecTDMatMultExprTrait< VT, TDMatTSMatMultExpr<MT1,MT2> >
1966 {
1967  public:
1968  //**********************************************************************************************
1969  using Type = If_< And< IsSparseVector<VT>, IsRowVector<VT>
1970  , IsDenseMatrix<MT1>, IsColumnMajorMatrix<MT1>
1971  , IsSparseMatrix<MT2>, IsColumnMajorMatrix<MT2> >
1972  , TDVecTSMatMultExprTrait_< TSVecTDMatMultExprTrait_<VT,MT1>, MT2 >
1973  , INVALID_TYPE >;
1974  //**********************************************************************************************
1975 };
1977 //*************************************************************************************************
1978 
1979 
1980 //*************************************************************************************************
1982 template< typename MT1, typename MT2, bool AF >
1983 struct SubmatrixExprTrait< TDMatTSMatMultExpr<MT1,MT2>, AF >
1984 {
1985  public:
1986  //**********************************************************************************************
1987  using Type = MultExprTrait_< SubmatrixExprTrait_<const MT1,AF>
1988  , SubmatrixExprTrait_<const MT2,AF> >;
1989  //**********************************************************************************************
1990 };
1992 //*************************************************************************************************
1993 
1994 
1995 //*************************************************************************************************
1997 template< typename MT1, typename MT2 >
1998 struct RowExprTrait< TDMatTSMatMultExpr<MT1,MT2> >
1999 {
2000  public:
2001  //**********************************************************************************************
2002  using Type = MultExprTrait_< RowExprTrait_<const MT1>, MT2 >;
2003  //**********************************************************************************************
2004 };
2006 //*************************************************************************************************
2007 
2008 
2009 //*************************************************************************************************
2011 template< typename MT1, typename MT2 >
2012 struct ColumnExprTrait< TDMatTSMatMultExpr<MT1,MT2> >
2013 {
2014  public:
2015  //**********************************************************************************************
2016  using Type = MultExprTrait_< MT1, ColumnExprTrait_<const MT2> >;
2017  //**********************************************************************************************
2018 };
2020 //*************************************************************************************************
2021 
2022 } // namespace blaze
2023 
2024 #endif
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TDMatTSMatMultExpr.h:384
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TDMatTSMatMultExpr.h:416
Header file for auxiliary alias declarations.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:72
Header file for mathematical functions.
constexpr bool useOptimizedKernels
Configuration switch for optimized kernels.This configuration switch enables/disables all optimized c...
Definition: Optimizations.h:84
TDMatTSMatMultExpr< MT1, MT2 > This
Type of this TDMatTSMatMultExpr instance.
Definition: TDMatTSMatMultExpr.h:218
Header file for the Rows type trait.
Header file for the IsUniUpper type trait.
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:7800
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:87
Header file for basic type definitions.
Expression object for transpose dense matrix-transpose sparse matrix multiplications.The TDMatTSMatMultExpr class represents the compile time expression for multiplications between a column-major dense matrix and a column-major sparse matrix.
Definition: Forward.h:135
EnableIf_< IsDenseMatrix< MT1 > > smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
Header file for the IsSparseMatrix type trait.
Header file for the serial shim.
ResultType_< MT1 > RT1
Result type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:135
Header file for the IsDiagonal type trait.
BLAZE_ALWAYS_INLINE size_t size(const Vector< VT, TF > &vector) noexcept
Returns the current size/dimension of the vector.
Definition: Vector.h:258
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the ColumnExprTrait class template.
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:188
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:162
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:315
Header file for the IsColumnMajorMatrix type trait.
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:533
Header file for the IsRowVector type trait.
Header file for the And class template.
const ElementType_< MT > min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1669
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:88
OppositeType_< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:220
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:162
MultTrait_< RT1, RT2 > ResultType
Result type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:219
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:224
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:723
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:245
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:88
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Header file for the IsUniLower type trait.
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:323
const ElementType_< MT > max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1716
EnableIf_< IsDenseMatrix< MT1 > > smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
DisableIf_< Or< IsComputation< MT >, IsTransExpr< MT > >, ColumnExprTrait_< MT > > column(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific column of the given matrix.
Definition: Column.h:126
CompositeType_< MT2 > CT2
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:140
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:109
ResultType_< MT2 > RT2
Result type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:136
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:573
Constraint on the data type.
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: TDMatTSMatMultExpr.h:326
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatTSMatMultExpr.h:225
Constraint on the data type.
Constraint on the data type.
Header file for the MultExprTrait class template.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:72
SubvectorExprTrait_< VT, unaligned > subvector(Vector< VT, TF > &vector, size_t index, size_t size)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:152
IfTrue_< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatTSMatMultExpr.h:234
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TDMatTSMatMultExpr.h:396
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
Header file for the If class template.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
Header file for the TSVecTDMatMultExprTrait class template.
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2647
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Header file for the Or class template.
Header file for the TDMatSVecMultExprTrait class template.
Header file for the TDVecTSMatMultExprTrait class template.
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
Header file for the DenseMatrix base class.
Header file for the Columns type trait.
Header file for the TSMatDVecMultExprTrait class template.
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
Header file for the IsLower type trait.
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:90
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:60
Header file for the IsTriangular type trait.
Constraints on the storage order of matrix types.
Header file for the exception macros of the math module.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
CompositeType_< MT1 > CT1
Composite type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:139
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:254
Header file for the RowExprTrait class template.
Header file for all forward declarations for expression class templates.
Header file for the IsDenseMatrix type trait.
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatTSMatMultExpr.h:423
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:109
BLAZE_ALWAYS_INLINE const EnableIf_< And< IsIntegral< T >, HasSize< T, 1UL > >, If_< IsSigned< T >, SIMDint8, SIMDuint8 > > set(T value) noexcept
Sets all values in the vector to the given 1-byte integral value.
Definition: Set.h:76
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: TDMatTSMatMultExpr.h:342
DisableIf_< Or< IsComputation< MT >, IsTransExpr< MT > >, RowExprTrait_< MT > > row(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific row of the given matrix.
Definition: Row.h:126
Header file for the IsSparseVector type trait.
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
Header file for run time assertion macros.
Compile time check for column-major matrix types.This type trait tests whether or not the given templ...
Definition: IsColumnMajorMatrix.h:83
Utility type for generic codes.
ElementType_< RT1 > ET1
Element type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:137
If_< IsExpression< MT2 >, const MT2, const MT2 & > RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:231
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:160
RightOperand rightOperand() const noexcept
Returns the right-hand side transpose sparse matrix operand.
Definition: TDMatTSMatMultExpr.h:372
ElementType_< RT2 > ET2
Element type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:138
Header file for the reset shim.
Header file for the isDefault shim.
Constraints on the storage order of matrix types.
IntegralConstant< bool, B > BoolConstant
Generic wrapper for a compile time constant boolean value.The BoolConstant class template represents ...
Definition: IntegralConstant.h:100
Header file for the RemoveReference type trait.
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:223
typename T::OppositeType OppositeType_
Alias declaration for nested OppositeType type definitions.The OppositeType_ alias declaration provid...
Definition: Aliases.h:243
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:84
Header file for the IsDenseVector type trait.
IfTrue_< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: TDMatTSMatMultExpr.h:237
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
Header file for the AreSIMDCombinable type trait.
LeftOperand leftOperand() const noexcept
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatTSMatMultExpr.h:362
Header file for the IsComputation type trait class.
Compile time logical or evaluation.The Or class template performs at compile time a logical or ('&&')...
Definition: Or.h:101
TDMatTSMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the TDMatTSMatMultExpr class.
Definition: TDMatTSMatMultExpr.h:263
Header file for the TDMatDVecMultExprTrait class template.
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: TDMatTSMatMultExpr.h:278
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
If_< IsExpression< MT1 >, const MT1, const MT1 & > LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:228
Header file for the IntegralConstant class template.
ElementType_< ResultType > ElementType
Resulting element type.
Definition: TDMatTSMatMultExpr.h:222
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:403
Header file for the IsUpper type trait.
Header file for the IsColumnVector type trait.
Constraint on the data type.
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:221
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: TDMatTSMatMultExpr.h:424
Header file for the IsResizable type trait.
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: TDMatTSMatMultExpr.h:223
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the TDVecTDMatMultExprTrait class template.
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatTSMatMultExpr.h:406
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: TDMatTSMatMultExpr.h:352
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:61
Header file for the IsExpression type trait class.
Header file for the TSMatSVecMultExprTrait class template.
Header file for the FunctionTrace class.