TDMatTSMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <stdexcept>
52 #include <blaze/math/Intrinsics.h>
54 #include <blaze/math/shims/Reset.h>
92 #include <blaze/util/Assert.h>
94 #include <blaze/util/DisableIf.h>
95 #include <blaze/util/EnableIf.h>
96 #include <blaze/util/InvalidType.h>
98 #include <blaze/util/mpl/And.h>
99 #include <blaze/util/mpl/Or.h>
100 #include <blaze/util/SelectType.h>
101 #include <blaze/util/Types.h>
104 
105 
106 namespace blaze {
107 
108 //=================================================================================================
109 //
110 // CLASS TDMATTSMATMULTEXPR
111 //
112 //=================================================================================================
113 
114 //*************************************************************************************************
121 template< typename MT1 // Type of the left-hand side dense matrix
122  , typename MT2 > // Type of the right-hand side sparse matrix
123 class TDMatTSMatMultExpr : public DenseMatrix< TDMatTSMatMultExpr<MT1,MT2>, true >
124  , private MatMatMultExpr
125  , private Computation
126 {
127  private:
128  //**Type definitions****************************************************************************
129  typedef typename MT1::ResultType RT1;
130  typedef typename MT2::ResultType RT2;
131  typedef typename RT1::ElementType ET1;
132  typedef typename RT2::ElementType ET2;
133  typedef typename MT1::CompositeType CT1;
134  typedef typename MT2::CompositeType CT2;
135  //**********************************************************************************************
136 
137  //**********************************************************************************************
140  //**********************************************************************************************
141 
142  //**********************************************************************************************
144  enum { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
145  //**********************************************************************************************
146 
147  //**********************************************************************************************
149 
155  template< typename T1, typename T2, typename T3 >
156  struct CanExploitSymmetry {
157  enum { value = IsRowMajorMatrix<T1>::value &&
158  ( IsSymmetric<T2>::value || IsSymmetric<T3>::value ) };
159  };
161  //**********************************************************************************************
162 
163  //**********************************************************************************************
165 
169  template< typename T1, typename T2, typename T3 >
170  struct IsEvaluationRequired {
171  enum { value = ( evaluateLeft || evaluateRight ) &&
172  !CanExploitSymmetry<T1,T2,T3>::value };
173  };
175  //**********************************************************************************************
176 
177  //**********************************************************************************************
179 
182  template< typename T1, typename T2, typename T3 >
183  struct UseVectorizedKernel {
184  enum { value = !IsDiagonal<T2>::value &&
185  T1::vectorizable && T2::vectorizable &&
186  IsColumnMajorMatrix<T1>::value &&
187  IsSame<typename T1::ElementType,typename T2::ElementType>::value &&
188  IsSame<typename T1::ElementType,typename T3::ElementType>::value &&
189  IntrinsicTrait<typename T1::ElementType>::addition &&
190  IntrinsicTrait<typename T1::ElementType>::subtraction &&
191  IntrinsicTrait<typename T1::ElementType>::multiplication };
192  };
194  //**********************************************************************************************
195 
196  //**********************************************************************************************
198 
202  template< typename T1, typename T2, typename T3 >
203  struct UseOptimizedKernel {
204  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
205  !IsDiagonal<T2>::value &&
206  !IsResizable<typename T1::ElementType>::value &&
207  !IsResizable<ET2>::value };
208  };
210  //**********************************************************************************************
211 
212  //**********************************************************************************************
214 
217  template< typename T1, typename T2, typename T3 >
218  struct UseDefaultKernel {
219  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
220  !UseOptimizedKernel<T1,T2,T3>::value };
221  };
223  //**********************************************************************************************
224 
225  public:
226  //**Type definitions****************************************************************************
233  typedef const ElementType ReturnType;
234  typedef const ResultType CompositeType;
235 
237  typedef typename SelectType< IsExpression<MT1>::value, const MT1, const MT1& >::Type LeftOperand;
238 
240  typedef typename SelectType< IsExpression<MT2>::value, const MT2, const MT2& >::Type RightOperand;
241 
244 
247  //**********************************************************************************************
248 
249  //**Compilation flags***************************************************************************
251  enum { vectorizable = !IsDiagonal<MT1>::value &&
252  MT1::vectorizable &&
256 
258  enum { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
259  !evaluateRight && MT2::smpAssignable };
260  //**********************************************************************************************
261 
262  //**Constructor*********************************************************************************
268  explicit inline TDMatTSMatMultExpr( const MT1& lhs, const MT2& rhs )
269  : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
270  , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
271  {
272  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
273  }
274  //**********************************************************************************************
275 
276  //**Access operator*****************************************************************************
283  inline ReturnType operator()( size_t i, size_t j ) const {
284  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
285  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
286 
288 
289  ElementType tmp = ElementType();
290 
291  // Early exit
292  if( lhs_.columns() == 0UL )
293  return tmp;
294 
295  // Fast computation in case the right-hand side sparse matrix directly provides iterators
297  {
298  CT2 B( rhs_ ); // Evaluation of the right-hand side sparse matrix operand
299 
300  const ConstIterator end( ( IsLower<MT1>::value )
301  ?( IsStrictlyLower<MT1>::value ? B.lowerBound(i,j) : B.upperBound(i,j) )
302  :( B.end(j) ) );
303  ConstIterator element( ( IsUpper<MT1>::value )
304  ?( IsStrictlyUpper<MT1>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
305  :( B.begin(j) ) );
306 
307  if( element != end ) {
308  tmp = lhs_(i,element->index()) * element->value();
309  ++element;
310  for( ; element!=end; ++element ) {
311  tmp += lhs_(i,element->index()) * element->value();
312  }
313  }
314  }
315 
316  // Default computation in case the right-hand side sparse matrix doesn't provide iterators
317  else
318  {
319  const size_t kbegin( ( IsUpper<MT1>::value )
320  ?( ( IsLower<MT2>::value )
321  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
322  , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
323  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
324  :( ( IsLower<MT2>::value )
325  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
326  :( 0UL ) ) );
327  const size_t kend( ( IsLower<MT1>::value )
328  ?( ( IsUpper<MT2>::value )
329  ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
330  , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
331  :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
332  :( ( IsUpper<MT2>::value )
333  ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
334  :( lhs_.columns() ) ) );
335 
336  if( ( !IsTriangular<MT1>::value && !IsTriangular<MT2>::value ) || kbegin < kend ) {
337  tmp = lhs_(i,kbegin) * rhs_(kbegin,j);
338  for( size_t k=kbegin+1UL; k<kend; ++k ) {
339  tmp += lhs_(i,k) * rhs_(k,j);
340  }
341  }
342  }
343 
344  return tmp;
345  }
346  //**********************************************************************************************
347 
348  //**Rows function*******************************************************************************
353  inline size_t rows() const {
354  return lhs_.rows();
355  }
356  //**********************************************************************************************
357 
358  //**Columns function****************************************************************************
363  inline size_t columns() const {
364  return rhs_.columns();
365  }
366  //**********************************************************************************************
367 
368  //**Left operand access*************************************************************************
373  inline LeftOperand leftOperand() const {
374  return lhs_;
375  }
376  //**********************************************************************************************
377 
378  //**Right operand access************************************************************************
383  inline RightOperand rightOperand() const {
384  return rhs_;
385  }
386  //**********************************************************************************************
387 
388  //**********************************************************************************************
394  template< typename T >
395  inline bool canAlias( const T* alias ) const {
396  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
397  }
398  //**********************************************************************************************
399 
400  //**********************************************************************************************
406  template< typename T >
407  inline bool isAliased( const T* alias ) const {
408  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
409  }
410  //**********************************************************************************************
411 
412  //**********************************************************************************************
417  inline bool isAligned() const {
418  return lhs_.isAligned();
419  }
420  //**********************************************************************************************
421 
422  //**********************************************************************************************
427  inline bool canSMPAssign() const {
428  return ( columns() > SMP_TDMATTSMATMULT_THRESHOLD );
429  }
430  //**********************************************************************************************
431 
432  private:
433  //**Member variables****************************************************************************
434  LeftOperand lhs_;
435  RightOperand rhs_;
436  //**********************************************************************************************
437 
438  //**Assignment to dense matrices****************************************************************
451  template< typename MT // Type of the target dense matrix
452  , bool SO > // Storage order of the target dense matrix
453  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
454  assign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
455  {
457 
458  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
459  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
460 
461  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
462  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
463 
464  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
465  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
466  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
467  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
468  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
469  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
470 
471  TDMatTSMatMultExpr::selectAssignKernel( ~lhs, A, B );
472  }
474  //**********************************************************************************************
475 
476  //**Default assignment to row-major dense matrices**********************************************
490  template< typename MT3 // Type of the left-hand side target matrix
491  , typename MT4 // Type of the left-hand side matrix operand
492  , typename MT5 > // Type of the right-hand side matrix operand
493  static inline void selectAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
494  {
495  typedef typename MT5::ConstIterator ConstIterator;
496 
497  const size_t ipos( A.rows() & size_t(-4) );
498  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == ipos, "Invalid end calculation" );
499 
500  for( size_t i=0UL; i<ipos; i+=4UL ) {
501  for( size_t j=0UL; j<B.columns(); ++j )
502  {
503  ConstIterator element( ( IsUpper<MT4>::value )
504  ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
505  :( B.begin(j) ) );
506  const ConstIterator end( ( IsLower<MT4>::value )
507  ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
508  :( B.end(j) ) );
509 
510  if( element == end ) {
511  reset( (~C)(i ,j) );
512  reset( (~C)(i+1UL,j) );
513  reset( (~C)(i+2UL,j) );
514  reset( (~C)(i+3UL,j) );
515  continue;
516  }
517 
518  (~C)(i ,j) = A(i ,element->index()) * element->value();
519  (~C)(i+1UL,j) = A(i+1UL,element->index()) * element->value();
520  (~C)(i+2UL,j) = A(i+2UL,element->index()) * element->value();
521  (~C)(i+3UL,j) = A(i+3UL,element->index()) * element->value();
522  ++element;
523  for( ; element!=end; ++element ) {
524  (~C)(i ,j) += A(i ,element->index()) * element->value();
525  (~C)(i+1UL,j) += A(i+1UL,element->index()) * element->value();
526  (~C)(i+2UL,j) += A(i+2UL,element->index()) * element->value();
527  (~C)(i+3UL,j) += A(i+3UL,element->index()) * element->value();
528  }
529  }
530  }
531 
532  for( size_t i=ipos; i<A.rows(); ++i ) {
533  for( size_t j=0UL; j<B.columns(); ++j )
534  {
535  ConstIterator element( ( IsUpper<MT4>::value )
536  ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
537  :( B.begin(j) ) );
538  const ConstIterator end( ( IsLower<MT4>::value )
539  ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i,j) : B.upperBound(i,j) )
540  :( B.end(j) ) );
541 
542  if( element == end ) {
543  reset( (~C)(i,j) );
544  continue;
545  }
546 
547  (~C)(i,j) = A(i,element->index()) * element->value();
548  ++element;
549  for( ; element!=end; ++element )
550  (~C)(i,j) += A(i,element->index()) * element->value();
551  }
552  }
553  }
555  //**********************************************************************************************
556 
557  //**Default assignment to column-major dense matrices*******************************************
571  template< typename MT3 // Type of the left-hand side target matrix
572  , typename MT4 // Type of the left-hand side matrix operand
573  , typename MT5 > // Type of the right-hand side matrix operand
574  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
575  selectAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
576  {
577  typedef typename MT5::ConstIterator ConstIterator;
578 
579  for( size_t j=0UL; j<B.columns(); ++j )
580  {
581  for( size_t i=0UL; i<(~C).rows(); ++i ) {
582  reset( (~C)(i,j) );
583  }
584 
585  ConstIterator element( B.begin(j) );
586  const ConstIterator end( B.end(j) );
587 
588  for( ; element!=end; ++element )
589  {
590  const size_t j1( element->index() );
591 
592  if( IsDiagonal<MT4>::value )
593  {
594  (~C)(j1,j) = A(j1,j1) * element->value();
595  }
596  else
597  {
598  const size_t ibegin( ( IsLower<MT4>::value )
599  ?( IsStrictlyLower<MT4>::value ? j1+1UL : j1 )
600  :( 0UL ) );
601  const size_t iend( ( IsUpper<MT4>::value )
602  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
603  :( A.rows() ) );
604  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
605 
606  for( size_t i=ibegin; i<iend; ++i ) {
607  if( isDefault( (~C)(i,j) ) )
608  (~C)(i,j) = A(i,j1) * element->value();
609  else
610  (~C)(i,j) += A(i,j1) * element->value();
611  }
612  }
613  }
614  }
615  }
617  //**********************************************************************************************
618 
619  //**Optimized assignment to column-major dense matrices*****************************************
633  template< typename MT3 // Type of the left-hand side target matrix
634  , typename MT4 // Type of the left-hand side matrix operand
635  , typename MT5 > // Type of the right-hand side matrix operand
636  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
637  selectAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
638  {
639  typedef typename MT5::ConstIterator ConstIterator;
640 
641  reset( ~C );
642 
643  for( size_t j=0UL; j<B.columns(); ++j )
644  {
645  const ConstIterator end( B.end(j) );
646  ConstIterator element( B.begin(j) );
647 
648  const size_t nonzeros( B.nonZeros(j) );
649  const size_t kpos( nonzeros & size_t(-4) );
650  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
651 
652  for( size_t k=0UL; k<kpos; k+=4UL )
653  {
654  const size_t j1( element->index() );
655  const ET2 v1( element->value() );
656  ++element;
657  const size_t j2( element->index() );
658  const ET2 v2( element->value() );
659  ++element;
660  const size_t j3( element->index() );
661  const ET2 v3( element->value() );
662  ++element;
663  const size_t j4( element->index() );
664  const ET2 v4( element->value() );
665  ++element;
666 
667  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
668 
669  const size_t ibegin( ( IsLower<MT4>::value )
670  ?( IsStrictlyLower<MT4>::value ? j1+1UL : j1 )
671  :( 0UL ) );
672  const size_t iend( ( IsUpper<MT4>::value )
673  ?( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL )
674  :( A.rows() ) );
675  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
676 
677  const size_t inum( iend - ibegin );
678  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
679  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
680 
681  for( size_t i=ibegin; i<ipos; i+=4UL ) {
682  (~C)(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
683  (~C)(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
684  (~C)(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
685  (~C)(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
686  }
687  for( size_t i=ipos; i<iend; ++i ) {
688  (~C)(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
689  }
690  }
691 
692  for( ; element!=end; ++element )
693  {
694  const size_t j1( element->index() );
695  const ET2 v1( element->value() );
696 
697  const size_t ibegin( ( IsLower<MT4>::value )
698  ?( IsStrictlyLower<MT4>::value ? j1+1UL : j1 )
699  :( 0UL ) );
700  const size_t iend( ( IsUpper<MT4>::value )
701  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
702  :( A.rows() ) );
703  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
704 
705  const size_t inum( iend - ibegin );
706  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
707  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
708 
709  for( size_t i=ibegin; i<ipos; i+=4UL ) {
710  (~C)(i ,j) += A(i ,j1) * v1;
711  (~C)(i+1UL,j) += A(i+1UL,j1) * v1;
712  (~C)(i+2UL,j) += A(i+2UL,j1) * v1;
713  (~C)(i+3UL,j) += A(i+3UL,j1) * v1;
714  }
715  for( size_t i=ipos; i<iend; ++i ) {
716  (~C)(i,j) += A(i,j1) * v1;
717  }
718  }
719  }
720  }
722  //**********************************************************************************************
723 
724  //**Vectorized assignment to column-major dense matrices****************************************
738  template< typename MT3 // Type of the left-hand side target matrix
739  , typename MT4 // Type of the left-hand side matrix operand
740  , typename MT5 > // Type of the right-hand side matrix operand
741  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
742  selectAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
743  {
744  typedef IntrinsicTrait<ElementType> IT;
745  typedef typename MT5::ConstIterator ConstIterator;
746 
747  reset( ~C );
748 
749  for( size_t j=0UL; j<B.columns(); ++j )
750  {
751  const ConstIterator end( B.end(j) );
752  ConstIterator element( B.begin(j) );
753 
754  const size_t nonzeros( B.nonZeros(j) );
755  const size_t kpos( nonzeros & size_t(-4) );
756  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
757 
758  for( size_t k=0UL; k<kpos; k+=4UL )
759  {
760  const size_t j1( element->index() );
761  const IntrinsicType v1( set( element->value() ) );
762  ++element;
763  const size_t j2( element->index() );
764  const IntrinsicType v2( set( element->value() ) );
765  ++element;
766  const size_t j3( element->index() );
767  const IntrinsicType v3( set( element->value() ) );
768  ++element;
769  const size_t j4( element->index() );
770  const IntrinsicType v4( set( element->value() ) );
771  ++element;
772 
773  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
774 
775  const size_t ibegin( ( IsLower<MT4>::value )
776  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
777  :( 0UL ) );
778  const size_t iend( ( IsUpper<MT4>::value )
779  ?( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL )
780  :( A.rows() ) );
781  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
782 
783  for( size_t i=ibegin; i<iend; i+=IT::size ) {
784  (~C).store( i, j, (~C).load(i,j) + A.load(i,j1) * v1 + A.load(i,j2) * v2 + A.load(i,j3) * v3 + A.load(i,j4) * v4 );
785  }
786  }
787 
788  for( ; element!=end; ++element )
789  {
790  const size_t j1( element->index() );
791  const IntrinsicType v1( set( element->value() ) );
792 
793  const size_t ibegin( ( IsLower<MT4>::value )
794  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
795  :( 0UL ) );
796  const size_t iend( ( IsUpper<MT4>::value )
797  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
798  :( A.rows() ) );
799  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
800 
801  for( size_t i=ibegin; i<iend; i+=IT::size ) {
802  (~C).store( i, j, (~C).load(i,j) + A.load(i,j1) * v1 );
803  }
804  }
805  }
806  }
808  //**********************************************************************************************
809 
810  //**Assignment to sparse matrices***************************************************************
823  template< typename MT // Type of the target sparse matrix
824  , bool SO > // Storage order of the target sparse matrix
825  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
826  assign( SparseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
827  {
829 
830  typedef typename SelectType< SO, ResultType, OppositeType >::Type TmpType;
831 
838 
839  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
840  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
841 
842  const TmpType tmp( serial( rhs ) );
843  assign( ~lhs, tmp );
844  }
846  //**********************************************************************************************
847 
848  //**Restructuring assignment to row-major matrices************************************************************
863  template< typename MT > // Type of the target matrix
864  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
865  assign( Matrix<MT,false>& lhs, const TDMatTSMatMultExpr& rhs )
866  {
868 
870 
871  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
872  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
873 
874  if( IsSymmetric<MT1>::value && IsSymmetric<MT2>::value )
875  assign( ~lhs, trans( rhs.lhs_ ) * trans( rhs.rhs_ ) );
876  else if( IsSymmetric<MT1>::value )
877  assign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
878  else
879  assign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
880  }
882  //**********************************************************************************************
883 
884  //**Addition assignment to dense matrices*******************************************************
897  template< typename MT // Type of the target dense matrix
898  , bool SO > // Storage order of the target dense matrix
899  friend inline typename DisableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
900  addAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
901  {
903 
904  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
905  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
906 
907  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
908  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
909 
910  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
911  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
912  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
913  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
914  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
915  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
916 
917  TDMatTSMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
918  }
920  //**********************************************************************************************
921 
922  //**Default addition assignment to row-major dense matrices*************************************
936  template< typename MT3 // Type of the left-hand side target matrix
937  , typename MT4 // Type of the left-hand side matrix operand
938  , typename MT5 > // Type of the right-hand side matrix operand
939  static inline void selectAddAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
940  {
941  typedef typename MT5::ConstIterator ConstIterator;
942 
943  const size_t ipos( A.rows() & size_t(-4) );
944  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == ipos, "Invalid end calculation" );
945 
946  for( size_t i=0UL; i<ipos; i+=4UL ) {
947  for( size_t j=0UL; j<B.columns(); ++j )
948  {
949  ConstIterator element( ( IsUpper<MT4>::value )
950  ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
951  :( B.begin(j) ) );
952  const ConstIterator end( ( IsLower<MT4>::value )
953  ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
954  :( B.end(j) ) );
955 
956  for( ; element!=end; ++element ) {
957  (~C)(i ,j) += A(i ,element->index()) * element->value();
958  (~C)(i+1UL,j) += A(i+1UL,element->index()) * element->value();
959  (~C)(i+2UL,j) += A(i+2UL,element->index()) * element->value();
960  (~C)(i+3UL,j) += A(i+3UL,element->index()) * element->value();
961  }
962  }
963  }
964 
965  for( size_t i=ipos; i<A.rows(); ++i ) {
966  for( size_t j=0UL; j<B.columns(); ++j )
967  {
968  ConstIterator element( ( IsUpper<MT4>::value )
969  ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
970  :( B.begin(j) ) );
971  const ConstIterator end( ( IsLower<MT4>::value )
972  ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i,j) : B.upperBound(i,j) )
973  :( B.end(j) ) );
974 
975  for( ; element!=end; ++element )
976  (~C)(i,j) += A(i,element->index()) * element->value();
977  }
978  }
979  }
981  //**********************************************************************************************
982 
983  //**Default addition assignment to column-major dense matrices**********************************
997  template< typename MT3 // Type of the left-hand side target matrix
998  , typename MT4 // Type of the left-hand side matrix operand
999  , typename MT5 > // Type of the right-hand side matrix operand
1000  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
1001  selectAddAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1002  {
1003  typedef typename MT5::ConstIterator ConstIterator;
1004 
1005  for( size_t j=0UL; j<B.columns(); ++j )
1006  {
1007  ConstIterator element( B.begin(j) );
1008  const ConstIterator end( B.end(j) );
1009 
1010  for( ; element!=end; ++element )
1011  {
1012  const size_t j1( element->index() );
1013 
1014  if( IsDiagonal<MT4>::value )
1015  {
1016  (~C)(j1,j) += A(j1,j1) * element->value();
1017  }
1018  else
1019  {
1020  const size_t ibegin( ( IsLower<MT4>::value )
1021  ?( IsStrictlyLower<MT4>::value ? j1+1UL : j1 )
1022  :( 0UL ) );
1023  const size_t iend( ( IsUpper<MT4>::value )
1024  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
1025  :( A.rows() ) );
1026  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1027 
1028  const size_t inum( iend - ibegin );
1029  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1030  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1031 
1032  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1033  (~C)(i ,j) += A(i ,j1) * element->value();
1034  (~C)(i+1UL,j) += A(i+1UL,j1) * element->value();
1035  (~C)(i+2UL,j) += A(i+2UL,j1) * element->value();
1036  (~C)(i+3UL,j) += A(i+3UL,j1) * element->value();
1037  }
1038  for( size_t i=ipos; i<iend; ++i ) {
1039  (~C)(i,j) += A(i,j1) * element->value();
1040  }
1041  }
1042  }
1043  }
1044  }
1046  //**********************************************************************************************
1047 
1048  //**Optimized addition assignment to column-major dense matrices********************************
1062  template< typename MT3 // Type of the left-hand side target matrix
1063  , typename MT4 // Type of the left-hand side matrix operand
1064  , typename MT5 > // Type of the right-hand side matrix operand
1065  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
1066  selectAddAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1067  {
1068  typedef typename MT5::ConstIterator ConstIterator;
1069 
1070  for( size_t j=0UL; j<B.columns(); ++j )
1071  {
1072  const ConstIterator end( B.end(j) );
1073  ConstIterator element( B.begin(j) );
1074 
1075  const size_t nonzeros( B.nonZeros(j) );
1076  const size_t kpos( nonzeros & size_t(-4) );
1077  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1078 
1079  for( size_t k=0UL; k<kpos; k+=4UL )
1080  {
1081  const size_t j1( element->index() );
1082  const ET2 v1( element->value() );
1083  ++element;
1084  const size_t j2( element->index() );
1085  const ET2 v2( element->value() );
1086  ++element;
1087  const size_t j3( element->index() );
1088  const ET2 v3( element->value() );
1089  ++element;
1090  const size_t j4( element->index() );
1091  const ET2 v4( element->value() );
1092  ++element;
1093 
1094  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1095 
1096  const size_t ibegin( ( IsLower<MT4>::value )
1097  ?( IsStrictlyLower<MT4>::value ? j1+1UL : j1 )
1098  :( 0UL ) );
1099  const size_t iend( ( IsUpper<MT4>::value )
1100  ?( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL )
1101  :( A.rows() ) );
1102  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1103 
1104  const size_t inum( iend - ibegin );
1105  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1106  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1107 
1108  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1109  (~C)(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1110  (~C)(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1111  (~C)(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1112  (~C)(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1113  }
1114  for( size_t i=ipos; i<iend; ++i ) {
1115  (~C)(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1116  }
1117  }
1118 
1119  for( ; element!=end; ++element )
1120  {
1121  const size_t j1( element->index() );
1122  const ET2 v1( element->value() );
1123 
1124  const size_t ibegin( ( IsLower<MT4>::value )
1125  ?( IsStrictlyLower<MT4>::value ? j1+1UL : j1 )
1126  :( 0UL ) );
1127  const size_t iend( ( IsUpper<MT4>::value )
1128  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
1129  :( A.rows() ) );
1130  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1131 
1132  const size_t inum( iend - ibegin );
1133  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1134  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1135 
1136  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1137  (~C)(i ,j) += A(i ,j1) * v1;
1138  (~C)(i+1UL,j) += A(i+1UL,j1) * v1;
1139  (~C)(i+2UL,j) += A(i+2UL,j1) * v1;
1140  (~C)(i+3UL,j) += A(i+3UL,j1) * v1;
1141  }
1142  for( size_t i=ipos; i<iend; ++i ) {
1143  (~C)(i,j) += A(i,j1) * v1;
1144  }
1145  }
1146  }
1147  }
1149  //**********************************************************************************************
1150 
1151  //**Vectorized addition assignment to column-major dense matrices*******************************
1165  template< typename MT3 // Type of the left-hand side target matrix
1166  , typename MT4 // Type of the left-hand side matrix operand
1167  , typename MT5 > // Type of the right-hand side matrix operand
1168  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
1169  selectAddAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1170  {
1171  typedef IntrinsicTrait<ElementType> IT;
1172  typedef typename MT5::ConstIterator ConstIterator;
1173 
1174  for( size_t j=0UL; j<B.columns(); ++j )
1175  {
1176  const ConstIterator end( B.end(j) );
1177  ConstIterator element( B.begin(j) );
1178 
1179  const size_t nonzeros( B.nonZeros(j) );
1180  const size_t kpos( nonzeros & size_t(-4) );
1181  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1182 
1183  for( size_t k=0UL; k<kpos; k+=4UL )
1184  {
1185  const size_t j1( element->index() );
1186  const IntrinsicType v1( set( element->value() ) );
1187  ++element;
1188  const size_t j2( element->index() );
1189  const IntrinsicType v2( set( element->value() ) );
1190  ++element;
1191  const size_t j3( element->index() );
1192  const IntrinsicType v3( set( element->value() ) );
1193  ++element;
1194  const size_t j4( element->index() );
1195  const IntrinsicType v4( set( element->value() ) );
1196  ++element;
1197 
1198  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1199 
1200  const size_t ibegin( ( IsLower<MT4>::value )
1201  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
1202  :( 0UL ) );
1203  const size_t iend( ( IsUpper<MT4>::value )
1204  ?( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL )
1205  :( A.rows() ) );
1206  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1207 
1208  for( size_t i=ibegin; i<iend; i+=IT::size ) {
1209  (~C).store( i, j, (~C).load(i,j) + A.load(i,j1) * v1 + A.load(i,j2) * v2 + A.load(i,j3) * v3 + A.load(i,j4) * v4 );
1210  }
1211  }
1212 
1213  for( ; element!=end; ++element )
1214  {
1215  const size_t j1( element->index() );
1216  const IntrinsicType v1( set( element->value() ) );
1217 
1218  const size_t ibegin( ( IsLower<MT4>::value )
1219  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
1220  :( 0UL ) );
1221  const size_t iend( ( IsUpper<MT4>::value )
1222  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
1223  :( A.rows() ) );
1224  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1225 
1226  for( size_t i=ibegin; i<iend; i+=IT::size ) {
1227  (~C).store( i, j, (~C).load(i,j) + A.load(i,j1) * v1 );
1228  }
1229  }
1230  }
1231  }
1233  //**********************************************************************************************
1234 
1235  //**Restructuring addition assignment to row-major matrices*************************************
1250  template< typename MT > // Type of the target matrix
1251  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1252  addAssign( Matrix<MT,false>& lhs, const TDMatTSMatMultExpr& rhs )
1253  {
1255 
1257 
1258  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1259  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1260 
1261  if( IsSymmetric<MT1>::value && IsSymmetric<MT2>::value )
1262  addAssign( ~lhs, trans( rhs.lhs_ ) * trans( rhs.rhs_ ) );
1263  else if( IsSymmetric<MT1>::value )
1264  addAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1265  else
1266  addAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1267  }
1269  //**********************************************************************************************
1270 
1271  //**Addition assignment to sparse matrices******************************************************
1272  // No special implementation for the addition assignment to sparse matrices.
1273  //**********************************************************************************************
1274 
1275  //**Subtraction assignment to dense matrices****************************************************
1288  template< typename MT // Type of the target dense matrix
1289  , bool SO > // Storage order of the target dense matrix
1290  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1291  {
1293 
1294  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1295  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1296 
1297  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1298  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1299 
1300  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1301  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1302  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1303  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1304  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1305  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1306 
1307  TDMatTSMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1308  }
1310  //**********************************************************************************************
1311 
1312  //**Default subtraction assignment to row-major dense matrices**********************************
1326  template< typename MT3 // Type of the left-hand side target matrix
1327  , typename MT4 // Type of the left-hand side matrix operand
1328  , typename MT5 > // Type of the right-hand side matrix operand
1329  static inline void selectSubAssignKernel( DenseMatrix<MT3,false>& C, const MT4& A, const MT5& B )
1330  {
1331  typedef typename MT5::ConstIterator ConstIterator;
1332 
1333  const size_t ipos( A.rows() & size_t(-4) );
1334  BLAZE_INTERNAL_ASSERT( ( A.rows() - ( A.rows() % 4UL ) ) == ipos, "Invalid end calculation" );
1335 
1336  for( size_t i=0UL; i<ipos; i+=4UL ) {
1337  for( size_t j=0UL; j<B.columns(); ++j )
1338  {
1339  ConstIterator element( ( IsUpper<MT4>::value )
1340  ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
1341  :( B.begin(j) ) );
1342  const ConstIterator end( ( IsLower<MT4>::value )
1343  ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
1344  :( B.end(j) ) );
1345 
1346  for( ; element!=end; ++element ) {
1347  (~C)(i ,j) -= A(i ,element->index()) * element->value();
1348  (~C)(i+1UL,j) -= A(i+1UL,element->index()) * element->value();
1349  (~C)(i+2UL,j) -= A(i+2UL,element->index()) * element->value();
1350  (~C)(i+3UL,j) -= A(i+3UL,element->index()) * element->value();
1351  }
1352  }
1353  }
1354 
1355  for( size_t i=ipos; i<A.rows(); ++i ) {
1356  for( size_t j=0UL; j<B.columns(); ++j )
1357  {
1358  ConstIterator element( ( IsUpper<MT4>::value )
1359  ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
1360  :( B.begin(j) ) );
1361  const ConstIterator end( ( IsLower<MT4>::value )
1362  ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i,j) : B.upperBound(i,j) )
1363  :( B.end(j) ) );
1364 
1365  for( ; element!=end; ++element )
1366  (~C)(i,j) -= A(i,element->index()) * element->value();
1367  }
1368  }
1369  }
1371  //**********************************************************************************************
1372 
1373  //**Default subtraction assignment to column-major dense matrices*******************************
1387  template< typename MT3 // Type of the left-hand side target matrix
1388  , typename MT4 // Type of the left-hand side matrix operand
1389  , typename MT5 > // Type of the right-hand side matrix operand
1390  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
1391  selectSubAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1392  {
1393  typedef typename MT5::ConstIterator ConstIterator;
1394 
1395  for( size_t j=0UL; j<B.columns(); ++j )
1396  {
1397  ConstIterator element( B.begin(j) );
1398  const ConstIterator end( B.end(j) );
1399 
1400  for( ; element!=end; ++element )
1401  {
1402  const size_t j1( element->index() );
1403 
1404  if( IsDiagonal<MT4>::value )
1405  {
1406  (~C)(j1,j) -= A(j1,j1) * element->value();
1407  }
1408  else
1409  {
1410  const size_t ibegin( ( IsLower<MT4>::value )
1411  ?( IsStrictlyLower<MT4>::value ? j1+1UL : j1 )
1412  :( 0UL ) );
1413  const size_t iend( ( IsUpper<MT4>::value )
1414  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
1415  :( A.rows() ) );
1416  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1417 
1418  const size_t inum( iend - ibegin );
1419  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1420  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1421 
1422  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1423  (~C)(i ,j) -= A(i ,j1) * element->value();
1424  (~C)(i+1UL,j) -= A(i+1UL,j1) * element->value();
1425  (~C)(i+2UL,j) -= A(i+2UL,j1) * element->value();
1426  (~C)(i+3UL,j) -= A(i+3UL,j1) * element->value();
1427  }
1428  for( size_t i=ipos; i<iend; ++i ) {
1429  (~C)(i,j) -= A(i,j1) * element->value();
1430  }
1431  }
1432  }
1433  }
1434  }
1436  //**********************************************************************************************
1437 
1438  //**Optimized subtraction assignment to column-major dense matrices*****************************
1452  template< typename MT3 // Type of the left-hand side target matrix
1453  , typename MT4 // Type of the left-hand side matrix operand
1454  , typename MT5 > // Type of the right-hand side matrix operand
1455  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
1456  selectSubAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1457  {
1458  typedef typename MT5::ConstIterator ConstIterator;
1459 
1460  for( size_t j=0UL; j<B.columns(); ++j )
1461  {
1462  const ConstIterator end( B.end(j) );
1463  ConstIterator element( B.begin(j) );
1464 
1465  const size_t nonzeros( B.nonZeros(j) );
1466  const size_t kpos( nonzeros & size_t(-4) );
1467  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1468 
1469  for( size_t k=0UL; k<kpos; k+=4UL )
1470  {
1471  const size_t j1( element->index() );
1472  const ET2 v1( element->value() );
1473  ++element;
1474  const size_t j2( element->index() );
1475  const ET2 v2( element->value() );
1476  ++element;
1477  const size_t j3( element->index() );
1478  const ET2 v3( element->value() );
1479  ++element;
1480  const size_t j4( element->index() );
1481  const ET2 v4( element->value() );
1482  ++element;
1483 
1484  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1485 
1486  const size_t ibegin( ( IsLower<MT4>::value )
1487  ?( IsStrictlyLower<MT4>::value ? j1+1UL : j1 )
1488  :( 0UL ) );
1489  const size_t iend( ( IsUpper<MT4>::value )
1490  ?( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL )
1491  :( A.rows() ) );
1492  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1493 
1494  const size_t inum( iend - ibegin );
1495  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1496  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1497 
1498  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1499  (~C)(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1500  (~C)(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1501  (~C)(i+2UL,j) -= A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1502  (~C)(i+3UL,j) -= A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1503  }
1504  for( size_t i=ipos; i<iend; ++i ) {
1505  (~C)(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1506  }
1507  }
1508 
1509  for( ; element!=end; ++element )
1510  {
1511  const size_t j1( element->index() );
1512  const ET2 v1( element->value() );
1513 
1514  const size_t ibegin( ( IsLower<MT4>::value )
1515  ?( IsStrictlyLower<MT4>::value ? j1+1UL : j1 )
1516  :( 0UL ) );
1517  const size_t iend( ( IsUpper<MT4>::value )
1518  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
1519  :( A.rows() ) );
1520  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1521 
1522  const size_t inum( iend - ibegin );
1523  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1524  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1525 
1526  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1527  (~C)(i ,j) -= A(i ,j1) * v1;
1528  (~C)(i+1UL,j) -= A(i+1UL,j1) * v1;
1529  (~C)(i+2UL,j) -= A(i+2UL,j1) * v1;
1530  (~C)(i+3UL,j) -= A(i+3UL,j1) * v1;
1531  }
1532  for( size_t i=ipos; i<iend; ++i ) {
1533  (~C)(i,j) -= A(i,j1) * v1;
1534  }
1535  }
1536  }
1537  }
1539  //**********************************************************************************************
1540 
1541  //**Vectorized subtraction assignment to column-major dense matrices****************************
1555  template< typename MT3 // Type of the left-hand side target matrix
1556  , typename MT4 // Type of the left-hand side matrix operand
1557  , typename MT5 > // Type of the right-hand side matrix operand
1558  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
1559  selectSubAssignKernel( DenseMatrix<MT3,true>& C, const MT4& A, const MT5& B )
1560  {
1561  typedef IntrinsicTrait<ElementType> IT;
1562  typedef typename MT5::ConstIterator ConstIterator;
1563 
1564  for( size_t j=0UL; j<B.columns(); ++j )
1565  {
1566  const ConstIterator end( B.end(j) );
1567  ConstIterator element( B.begin(j) );
1568 
1569  const size_t nonzeros( B.nonZeros(j) );
1570  const size_t kpos( nonzeros & size_t(-4) );
1571  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1572 
1573  for( size_t k=0UL; k<kpos; k+=4UL )
1574  {
1575  const size_t j1( element->index() );
1576  const IntrinsicType v1( set( element->value() ) );
1577  ++element;
1578  const size_t j2( element->index() );
1579  const IntrinsicType v2( set( element->value() ) );
1580  ++element;
1581  const size_t j3( element->index() );
1582  const IntrinsicType v3( set( element->value() ) );
1583  ++element;
1584  const size_t j4( element->index() );
1585  const IntrinsicType v4( set( element->value() ) );
1586  ++element;
1587 
1588  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1589 
1590  const size_t ibegin( ( IsLower<MT4>::value )
1591  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
1592  :( 0UL ) );
1593  const size_t iend( ( IsUpper<MT4>::value )
1594  ?( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL )
1595  :( A.rows() ) );
1596  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1597 
1598  for( size_t i=ibegin; i<iend; i+=IT::size ) {
1599  (~C).store( i, j, (~C).load(i,j) - A.load(i,j1) * v1 - A.load(i,j2) * v2 - A.load(i,j3) * v3 - A.load(i,j4) * v4 );
1600  }
1601  }
1602 
1603  for( ; element!=end; ++element )
1604  {
1605  const size_t j1( element->index() );
1606  const IntrinsicType v1( set( element->value() ) );
1607 
1608  const size_t ibegin( ( IsLower<MT4>::value )
1609  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
1610  :( 0UL ) );
1611  const size_t iend( ( IsUpper<MT4>::value )
1612  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
1613  :( A.rows() ) );
1614  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1615 
1616  for( size_t i=ibegin; i<iend; i+=IT::size ) {
1617  (~C).store( i, j, (~C).load(i,j) - A.load(i,j1) * v1 );
1618  }
1619  }
1620  }
1621  }
1623  //**********************************************************************************************
1624 
1625  //**Restructuring subtraction assignment to row-major matrices**********************************
1641  template< typename MT > // Type of the target matrix
1642  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1643  subAssign( Matrix<MT,false>& lhs, const TDMatTSMatMultExpr& rhs )
1644  {
1646 
1648 
1649  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1650  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1651 
1652  if( IsSymmetric<MT1>::value && IsSymmetric<MT2>::value )
1653  subAssign( ~lhs, trans( rhs.lhs_ ) * trans( rhs.rhs_ ) );
1654  else if( IsSymmetric<MT1>::value )
1655  subAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1656  else
1657  subAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1658  }
1660  //**********************************************************************************************
1661 
1662  //**Subtraction assignment to sparse matrices***************************************************
1663  // No special implementation for the subtraction assignment to sparse matrices.
1664  //**********************************************************************************************
1665 
1666  //**Multiplication assignment to dense matrices*************************************************
1667  // No special implementation for the multiplication assignment to dense matrices.
1668  //**********************************************************************************************
1669 
1670  //**Multiplication assignment to sparse matrices************************************************
1671  // No special implementation for the multiplication assignment to sparse matrices.
1672  //**********************************************************************************************
1673 
1674  //**SMP assignment to dense matrices************************************************************
1690  template< typename MT // Type of the target dense matrix
1691  , bool SO > // Storage order of the target dense matrix
1692  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1693  smpAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1694  {
1696 
1697  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1698  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1699 
1700  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1701  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1702 
1703  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1704  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1705  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1706  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1707  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1708  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1709 
1710  smpAssign( ~lhs, A * B );
1711  }
1713  //**********************************************************************************************
1714 
1715  //**SMP assignment to sparse matrices***********************************************************
1731  template< typename MT // Type of the target sparse matrix
1732  , bool SO > // Storage order of the target sparse matrix
1733  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1734  smpAssign( SparseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1735  {
1737 
1738  typedef typename SelectType< SO, ResultType, OppositeType >::Type TmpType;
1739 
1746 
1747  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1748  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1749 
1750  const TmpType tmp( rhs );
1751  smpAssign( ~lhs, tmp );
1752  }
1754  //**********************************************************************************************
1755 
1756  //**Restructuring SMP assignment to row-major matrices******************************************
1771  template< typename MT > // Type of the target matrix
1772  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1773  smpAssign( Matrix<MT,false>& lhs, const TDMatTSMatMultExpr& rhs )
1774  {
1776 
1778 
1779  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1780  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1781 
1782  if( IsSymmetric<MT1>::value && IsSymmetric<MT2>::value )
1783  smpAssign( ~lhs, trans( rhs.lhs_ ) * trans( rhs.rhs_ ) );
1784  else if( IsSymmetric<MT1>::value )
1785  smpAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1786  else
1787  smpAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1788  }
1790  //**********************************************************************************************
1791 
1792  //**SMP addition assignment to dense matrices***************************************************
1808  template< typename MT // Type of the target dense matrix
1809  , bool SO > // Storage order of the target dense matrix
1810  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1811  smpAddAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1812  {
1814 
1815  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1816  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1817 
1818  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1819  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1820 
1821  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1822  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1823  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1824  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1825  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1826  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1827 
1828  smpAddAssign( ~lhs, A * B );
1829  }
1831  //**********************************************************************************************
1832 
1833  //**Restructuring SMP addition assignment to row-major matrices*********************************
1849  template< typename MT > // Type of the target matrix
1850  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1851  smpAddAssign( Matrix<MT,false>& lhs, const TDMatTSMatMultExpr& rhs )
1852  {
1854 
1856 
1857  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1858  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1859 
1860  if( IsSymmetric<MT1>::value && IsSymmetric<MT2>::value )
1861  smpAddAssign( ~lhs, trans( rhs.lhs_ ) * trans( rhs.rhs_ ) );
1862  else if( IsSymmetric<MT1>::value )
1863  smpAddAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1864  else
1865  smpAddAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1866  }
1868  //**********************************************************************************************
1869 
1870  //**SMP addition assignment to sparse matrices**************************************************
1871  // No special implementation for the SMP addition assignment to sparse matrices.
1872  //**********************************************************************************************
1873 
1874  //**SMP subtraction assignment to dense matrices************************************************
1890  template< typename MT // Type of the target dense matrix
1891  , bool SO > // Storage order of the target dense matrix
1892  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1893  smpSubAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1894  {
1896 
1897  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1898  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1899 
1900  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1901  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1902 
1903  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1904  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1905  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1906  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1907  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1908  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1909 
1910  smpSubAssign( ~lhs, A * B );
1911  }
1913  //**********************************************************************************************
1914 
1915  //**Restructuring SMP subtraction assignment to row-major matrices******************************
1931  template< typename MT > // Type of the target matrix
1932  friend inline typename EnableIf< CanExploitSymmetry<MT,MT1,MT2> >::Type
1933  smpSubAssign( Matrix<MT,false>& lhs, const TDMatTSMatMultExpr& rhs )
1934  {
1936 
1938 
1939  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1940  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1941 
1942  if( IsSymmetric<MT1>::value && IsSymmetric<MT2>::value )
1943  smpSubAssign( ~lhs, trans( rhs.lhs_ ) * trans( rhs.rhs_ ) );
1944  else if( IsSymmetric<MT1>::value )
1945  smpSubAssign( ~lhs, trans( rhs.lhs_ ) * rhs.rhs_ );
1946  else
1947  smpSubAssign( ~lhs, rhs.lhs_ * trans( rhs.rhs_ ) );
1948  }
1950  //**********************************************************************************************
1951 
1952  //**SMP subtraction assignment to sparse matrices***********************************************
1953  // No special implementation for the SMP subtraction assignment to sparse matrices.
1954  //**********************************************************************************************
1955 
1956  //**SMP multiplication assignment to dense matrices*********************************************
1957  // No special implementation for the SMP multiplication assignment to dense matrices.
1958  //**********************************************************************************************
1959 
1960  //**SMP multiplication assignment to sparse matrices********************************************
1961  // No special implementation for the SMP multiplication assignment to sparse matrices.
1962  //**********************************************************************************************
1963 
1964  //**Compile time checks*************************************************************************
1972  //**********************************************************************************************
1973 };
1974 //*************************************************************************************************
1975 
1976 
1977 
1978 
1979 //=================================================================================================
1980 //
1981 // GLOBAL BINARY ARITHMETIC OPERATORS
1982 //
1983 //=================================================================================================
1984 
1985 //*************************************************************************************************
2014 template< typename T1 // Type of the left-hand side dense matrix
2015  , typename T2 > // Type of the right-hand side sparse matrix
2016 inline const TDMatTSMatMultExpr<T1,T2>
2018 {
2020 
2021  if( (~lhs).columns() != (~rhs).rows() )
2022  throw std::invalid_argument( "Matrix sizes do not match" );
2023 
2024  return TDMatTSMatMultExpr<T1,T2>( ~lhs, ~rhs );
2025 }
2026 //*************************************************************************************************
2027 
2028 
2029 
2030 
2031 //=================================================================================================
2032 //
2033 // ROWS SPECIALIZATIONS
2034 //
2035 //=================================================================================================
2036 
2037 //*************************************************************************************************
2039 template< typename MT1, typename MT2 >
2040 struct Rows< TDMatTSMatMultExpr<MT1,MT2> >
2041  : public Rows<MT1>
2042 {};
2044 //*************************************************************************************************
2045 
2046 
2047 
2048 
2049 //=================================================================================================
2050 //
2051 // COLUMNS SPECIALIZATIONS
2052 //
2053 //=================================================================================================
2054 
2055 //*************************************************************************************************
2057 template< typename MT1, typename MT2 >
2058 struct Columns< TDMatTSMatMultExpr<MT1,MT2> >
2059  : public Columns<MT2>
2060 {};
2062 //*************************************************************************************************
2063 
2064 
2065 
2066 
2067 //=================================================================================================
2068 //
2069 // ISLOWER SPECIALIZATIONS
2070 //
2071 //=================================================================================================
2072 
2073 //*************************************************************************************************
2075 template< typename MT1, typename MT2 >
2076 struct IsLower< TDMatTSMatMultExpr<MT1,MT2> >
2077  : public IsTrue< And< IsLower<MT1>, IsLower<MT2> >::value >
2078 {};
2080 //*************************************************************************************************
2081 
2082 
2083 
2084 
2085 //=================================================================================================
2086 //
2087 // ISUNILOWER SPECIALIZATIONS
2088 //
2089 //=================================================================================================
2090 
2091 //*************************************************************************************************
2093 template< typename MT1, typename MT2 >
2094 struct IsUniLower< TDMatTSMatMultExpr<MT1,MT2> >
2095  : public IsTrue< And< IsUniLower<MT1>, IsUniLower<MT2> >::value >
2096 {};
2098 //*************************************************************************************************
2099 
2100 
2101 
2102 
2103 //=================================================================================================
2104 //
2105 // ISSTRICTLYLOWER SPECIALIZATIONS
2106 //
2107 //=================================================================================================
2108 
2109 //*************************************************************************************************
2111 template< typename MT1, typename MT2 >
2112 struct IsStrictlyLower< TDMatTSMatMultExpr<MT1,MT2> >
2113  : public IsTrue< Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
2114  , And< IsStrictlyLower<MT2>, IsLower<MT1> > >::value >
2115 {};
2117 //*************************************************************************************************
2118 
2119 
2120 
2121 
2122 //=================================================================================================
2123 //
2124 // ISUPPER SPECIALIZATIONS
2125 //
2126 //=================================================================================================
2127 
2128 //*************************************************************************************************
2130 template< typename MT1, typename MT2 >
2131 struct IsUpper< TDMatTSMatMultExpr<MT1,MT2> >
2132  : public IsTrue< And< IsUpper<MT1>, IsUpper<MT2> >::value >
2133 {};
2135 //*************************************************************************************************
2136 
2137 
2138 
2139 
2140 //=================================================================================================
2141 //
2142 // ISUNIUPPER SPECIALIZATIONS
2143 //
2144 //=================================================================================================
2145 
2146 //*************************************************************************************************
2148 template< typename MT1, typename MT2 >
2149 struct IsUniUpper< TDMatTSMatMultExpr<MT1,MT2> >
2150  : public IsTrue< And< IsUniUpper<MT1>, IsUniUpper<MT2> >::value >
2151 {};
2153 //*************************************************************************************************
2154 
2155 
2156 
2157 
2158 //=================================================================================================
2159 //
2160 // ISSTRICTLYUPPER SPECIALIZATIONS
2161 //
2162 //=================================================================================================
2163 
2164 //*************************************************************************************************
2166 template< typename MT1, typename MT2 >
2167 struct IsStrictlyUpper< TDMatTSMatMultExpr<MT1,MT2> >
2168  : public IsTrue< Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
2169  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > >::value >
2170 {};
2172 //*************************************************************************************************
2173 
2174 
2175 
2176 
2177 //=================================================================================================
2178 //
2179 // EXPRESSION TRAIT SPECIALIZATIONS
2180 //
2181 //=================================================================================================
2182 
2183 //*************************************************************************************************
2185 template< typename MT1, typename MT2, typename VT >
2186 struct TDMatDVecMultExprTrait< TDMatTSMatMultExpr<MT1,MT2>, VT >
2187 {
2188  public:
2189  //**********************************************************************************************
2190  typedef typename SelectType< IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
2191  IsSparseMatrix<MT2>::value && IsColumnMajorMatrix<MT2>::value &&
2192  IsDenseVector<VT>::value && IsColumnVector<VT>::value
2193  , typename TDMatDVecMultExprTrait< MT1, typename TSMatDVecMultExprTrait<MT2,VT>::Type >::Type
2194  , INVALID_TYPE >::Type Type;
2195  //**********************************************************************************************
2196 };
2198 //*************************************************************************************************
2199 
2200 
2201 //*************************************************************************************************
2203 template< typename MT1, typename MT2, typename VT >
2204 struct TDMatSVecMultExprTrait< TDMatTSMatMultExpr<MT1,MT2>, VT >
2205 {
2206  public:
2207  //**********************************************************************************************
2208  typedef typename SelectType< IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
2209  IsSparseMatrix<MT2>::value && IsColumnMajorMatrix<MT2>::value &&
2210  IsSparseVector<VT>::value && IsColumnVector<VT>::value
2211  , typename TDMatDVecMultExprTrait< MT1, typename TSMatDVecMultExprTrait<MT2,VT>::Type >::Type
2212  , INVALID_TYPE >::Type Type;
2213  //**********************************************************************************************
2214 };
2216 //*************************************************************************************************
2217 
2218 
2219 //*************************************************************************************************
2221 template< typename VT, typename MT1, typename MT2 >
2222 struct TDVecTDMatMultExprTrait< VT, TDMatTSMatMultExpr<MT1,MT2> >
2223 {
2224  public:
2225  //**********************************************************************************************
2226  typedef typename SelectType< IsDenseVector<VT>::value && IsRowVector<VT>::value &&
2227  IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
2228  IsSparseMatrix<MT2>::value && IsColumnMajorMatrix<MT2>::value
2229  , typename TDVecTSMatMultExprTrait< typename TDVecTDMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
2230  , INVALID_TYPE >::Type Type;
2231  //**********************************************************************************************
2232 };
2234 //*************************************************************************************************
2235 
2236 
2237 //*************************************************************************************************
2239 template< typename VT, typename MT1, typename MT2 >
2240 struct TSVecTDMatMultExprTrait< VT, TDMatTSMatMultExpr<MT1,MT2> >
2241 {
2242  public:
2243  //**********************************************************************************************
2244  typedef typename SelectType< IsSparseVector<VT>::value && IsRowVector<VT>::value &&
2245  IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
2246  IsSparseMatrix<MT2>::value && IsColumnMajorMatrix<MT2>::value
2247  , typename TDVecTSMatMultExprTrait< typename TSVecTDMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
2248  , INVALID_TYPE >::Type Type;
2249  //**********************************************************************************************
2250 };
2252 //*************************************************************************************************
2253 
2254 
2255 //*************************************************************************************************
2257 template< typename MT1, typename MT2, bool AF >
2258 struct SubmatrixExprTrait< TDMatTSMatMultExpr<MT1,MT2>, AF >
2259 {
2260  public:
2261  //**********************************************************************************************
2262  typedef typename MultExprTrait< typename SubmatrixExprTrait<const MT1,AF>::Type
2263  , typename SubmatrixExprTrait<const MT2,AF>::Type >::Type Type;
2264  //**********************************************************************************************
2265 };
2267 //*************************************************************************************************
2268 
2269 
2270 //*************************************************************************************************
2272 template< typename MT1, typename MT2 >
2273 struct RowExprTrait< TDMatTSMatMultExpr<MT1,MT2> >
2274 {
2275  public:
2276  //**********************************************************************************************
2277  typedef typename MultExprTrait< typename RowExprTrait<const MT1>::Type, MT2 >::Type Type;
2278  //**********************************************************************************************
2279 };
2281 //*************************************************************************************************
2282 
2283 
2284 //*************************************************************************************************
2286 template< typename MT1, typename MT2 >
2287 struct ColumnExprTrait< TDMatTSMatMultExpr<MT1,MT2> >
2288 {
2289  public:
2290  //**********************************************************************************************
2291  typedef typename MultExprTrait< MT1, typename ColumnExprTrait<const MT2>::Type >::Type Type;
2292  //**********************************************************************************************
2293 };
2295 //*************************************************************************************************
2296 
2297 } // namespace blaze
2298 
2299 #endif
const MT::ElementType max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1649
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:89
TDMatTSMatMultExpr< MT1, MT2 > This
Type of this TDMatTSMatMultExpr instance.
Definition: TDMatTSMatMultExpr.h:227
Header file for the Rows type trait.
Header file for the IsUniUpper type trait.
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:8247
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:105
Header file for basic type definitions.
BLAZE_ALWAYS_INLINE size_t size(const Vector< VT, TF > &vector)
Returns the current size/dimension of the vector.
Definition: Vector.h:264
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:258
RT2::ElementType ET2
Element type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:132
Expression object for transpose dense matrix-transpose sparse matrix multiplications.The TDMatTSMatMultExpr class represents the compile time expression for multiplications between a column-major dense matrix and a column-major sparse matrix.
Definition: Forward.h:132
Header file for the IsSparseMatrix type trait.
Efficient implementation of a compressed matrix.The CompressedMatrix class template is the represent...
Definition: CompressedMatrix.h:209
Header file for the IsDiagonal type trait.
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:79
Header file for the ColumnExprTrait class template.
Header file for the IsColumnMajorMatrix type trait.
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:821
MT2::CompositeType CT2
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:134
MultTrait< RT1, RT2 >::Type ResultType
Result type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:228
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:2507
Header file for the IsRowVector type trait.
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:261
Header file for the And class template.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:90
SelectType< IsExpression< MT2 >::value, const MT2, const MT2 & >::Type RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:240
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:233
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:699
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Type relationship analysis.This class tests if the two data types A and B are equal. For this type comparison, the cv-qualifiers of both data types are ignored. If A and B are the same data type (ignoring the cv-qualifiers), then the value member enumeration is set to 1, the nested type definition Type is TrueType, and the class derives from TrueType. Otherwise value is set to 0, Type is FalseType, and the class derives from FalseType.
Definition: IsSame.h:158
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:90
Header file for the RequiresEvaluation type trait.
Header file for the IsUniLower type trait.
SelectType< evaluateRight, const RT2, CT2 >::Type RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: TDMatTSMatMultExpr.h:246
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:107
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:861
Constraint on the data type.
size_t columns() const
Returns the current number of columns of the matrix.
Definition: TDMatTSMatMultExpr.h:363
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatTSMatMultExpr.h:234
Constraint on the data type.
Constraint on the data type.
Header file for the MultExprTrait class template.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:90
SelectType< evaluateLeft, const RT1, CT1 >::Type LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatTSMatMultExpr.h:243
IntrinsicTrait< ElementType >::Type IntrinsicType
Resulting intrinsic element type.
Definition: TDMatTSMatMultExpr.h:232
const size_t SMP_TDMATTSMATMULT_THRESHOLD
SMP column-major dense matrix/column-major sparse matrix multiplication threshold.This threshold specifies when a column-major dense matrix/column-major sparse matrix multiplication can be executed in parallel. In case the number of rows/columns of the target matrix is larger or equal to this threshold, the operation is executed in parallel. If the number of rows/columns is below this threshold the operation is executed single-threaded.
Definition: Thresholds.h:995
Compile time type selection.The SelectType class template selects one of the two given types T1 and T...
Definition: SelectType.h:59
Header file for the DisableIf class template.
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
ResultType::TransposeType TransposeType
Transpose type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:230
Header file for the IsSymmetric type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: StorageOrder.h:161
Header file for the TSVecTDMatMultExprTrait class template.
TDMatTSMatMultExpr(const MT1 &lhs, const MT2 &rhs)
Constructor for the TDMatTSMatMultExpr class.
Definition: TDMatTSMatMultExpr.h:268
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2511
Header file for the Or class template.
BLAZE_ALWAYS_INLINE EnableIf< And< IsIntegral< T >, HasSize< T, 2UL > > >::Type store(T *address, const sse_int16_t &value)
Aligned store of a vector of 2-byte integral values.
Definition: Store.h:80
Header file for the TDMatSVecMultExprTrait class template.
Header file for the TDVecTSMatMultExprTrait class template.
const MT::ElementType min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1602
Header file for the DenseMatrix base class.
BLAZE_ALWAYS_INLINE void assign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the assignment of a matrix to a matrix.
Definition: Matrix.h:635
Header file for the Columns type trait.
Header file for the TSMatDVecMultExprTrait class template.
Header file for the IsLower type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:92
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:78
Header file for the IsTriangular type trait.
Constraints on the storage order of matrix types.
bool canAlias(const T *alias) const
Returns whether the expression can alias with the given address alias.
Definition: TDMatTSMatMultExpr.h:395
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:2505
Header file for the SelectType class template.
Header file for the RowExprTrait class template.
Header file for all forward declarations for expression class templates.
Header file for the IsDenseMatrix type trait.
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatTSMatMultExpr.h:434
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
Header file for the serial shim.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:165
LeftOperand leftOperand() const
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatTSMatMultExpr.h:373
ResultType::OppositeType OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:229
bool isAliased(const T *alias) const
Returns whether the expression is aliased with the given address alias.
Definition: TDMatTSMatMultExpr.h:407
EnableIf< IsDenseMatrix< MT1 > >::Type smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
Header file for the IsSparseVector type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type, a compilation error is created.
Definition: Symmetric.h:116
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: StorageOrder.h:81
Removal of reference modifiers.The RemoveCV type trait removes any reference modifiers from the given...
Definition: RemoveReference.h:69
size_t rows() const
Returns the current number of rows of the matrix.
Definition: TDMatTSMatMultExpr.h:353
Intrinsic characteristics of data types.The IntrinsicTrait class template provides the intrinsic char...
Definition: IntrinsicTrait.h:749
Header file for run time assertion macros.
EnableIf< IsDenseMatrix< MT1 > >::Type smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Utility type for generic codes.
bool canSMPAssign() const
Returns whether the expression can be used in SMP assignments.
Definition: TDMatTSMatMultExpr.h:427
Base template for the MultTrait class.
Definition: MultTrait.h:150
bool isAligned() const
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatTSMatMultExpr.h:417
BLAZE_ALWAYS_INLINE void addAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the addition assignment of a matrix to a matrix.
Definition: Matrix.h:742
Header file for the reset shim.
MT1::ResultType RT1
Result type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:129
Header file for the isDefault shim.
MT1::CompositeType CT1
Composite type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:133
RT1::ElementType ET1
Element type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:131
Header file for the RemoveReference type trait.
Substitution Failure Is Not An Error (SFINAE) class.The DisableIf class template is an auxiliary tool...
Definition: DisableIf.h:184
BLAZE_ALWAYS_INLINE EnableIf< And< IsIntegral< T >, HasSize< T, 2UL > >, sse_int16_t >::Type set(T value)
Sets all values in the vector to the given 2-byte integral value.
Definition: Set.h:73
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:283
Header file for the IsDenseVector type trait.
Header file for all intrinsic functionality.
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
ResultType::ElementType ElementType
Resulting element type.
Definition: TDMatTSMatMultExpr.h:231
Header file for the IsRowMajorMatrix type trait.
const DMatTransExpr< MT,!SO > trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:937
Header file for the IsComputation type trait class.
MT2::ResultType RT2
Result type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:130
EnableIf< IsDenseMatrix< MT1 > >::Type smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
Header file for the TDMatDVecMultExprTrait class template.
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: TDMatTSMatMultExpr.h:283
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
This ResultType
Result type for expression template evaluations.
Definition: CompressedMatrix.h:2502
Header file for the IsTrue value trait.
RightOperand rightOperand() const
Returns the right-hand side transpose sparse matrix operand.
Definition: TDMatTSMatMultExpr.h:383
Header file for the IsUpper type trait.
Header file for the IsColumnVector type trait.
Constraint on the data type.
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: TDMatTSMatMultExpr.h:435
Header file for the IsResizable type trait.
Size type of the Blaze library.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the TDVecTDMatMultExprTrait class template.
SelectType< IsExpression< MT1 >::value, const MT1, const MT1 & >::Type LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:237
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:79
Header file for the IsExpression type trait class.
Header file for the TSMatSVecMultExprTrait class template.
Header file for the FunctionTrace class.
BLAZE_ALWAYS_INLINE void subAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the subtraction assignment of a matrix to matrix.
Definition: Matrix.h:849