TDMatTSMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATTSMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
53 #include <blaze/math/Functions.h>
54 #include <blaze/math/Intrinsics.h>
56 #include <blaze/math/shims/Reset.h>
94 #include <blaze/util/Assert.h>
96 #include <blaze/util/EnableIf.h>
97 #include <blaze/util/Exception.h>
98 #include <blaze/util/InvalidType.h>
100 #include <blaze/util/mpl/And.h>
101 #include <blaze/util/mpl/Or.h>
102 #include <blaze/util/SelectType.h>
103 #include <blaze/util/Types.h>
106 
107 
108 namespace blaze {
109 
110 //=================================================================================================
111 //
112 // CLASS TDMATTSMATMULTEXPR
113 //
114 //=================================================================================================
115 
116 //*************************************************************************************************
123 template< typename MT1 // Type of the left-hand side dense matrix
124  , typename MT2 > // Type of the right-hand side sparse matrix
125 class TDMatTSMatMultExpr : public DenseMatrix< TDMatTSMatMultExpr<MT1,MT2>, true >
126  , private MatMatMultExpr
127  , private Computation
128 {
129  private:
130  //**Type definitions****************************************************************************
131  typedef typename MT1::ResultType RT1;
132  typedef typename MT2::ResultType RT2;
133  typedef typename RT1::ElementType ET1;
134  typedef typename RT2::ElementType ET2;
135  typedef typename MT1::CompositeType CT1;
136  typedef typename MT2::CompositeType CT2;
137  //**********************************************************************************************
138 
139  //**********************************************************************************************
142  //**********************************************************************************************
143 
144  //**********************************************************************************************
146  enum { evaluateRight = IsComputation<MT2>::value || RequiresEvaluation<MT2>::value };
147  //**********************************************************************************************
148 
149  //**********************************************************************************************
151 
155  template< typename T1, typename T2, typename T3 >
156  struct IsEvaluationRequired {
157  enum { value = ( evaluateLeft || evaluateRight ) };
158  };
160  //**********************************************************************************************
161 
162  //**********************************************************************************************
164 
167  template< typename T1, typename T2, typename T3 >
168  struct UseVectorizedKernel {
169  enum { value = useOptimizedKernels &&
170  !IsDiagonal<T2>::value &&
171  T1::vectorizable && T2::vectorizable &&
172  IsColumnMajorMatrix<T1>::value &&
173  IsSame<typename T1::ElementType,typename T2::ElementType>::value &&
174  IsSame<typename T1::ElementType,typename T3::ElementType>::value &&
175  IntrinsicTrait<typename T1::ElementType>::addition &&
176  IntrinsicTrait<typename T1::ElementType>::subtraction &&
177  IntrinsicTrait<typename T1::ElementType>::multiplication };
178  };
180  //**********************************************************************************************
181 
182  //**********************************************************************************************
184 
188  template< typename T1, typename T2, typename T3 >
189  struct UseOptimizedKernel {
190  enum { value = useOptimizedKernels &&
191  !UseVectorizedKernel<T1,T2,T3>::value &&
192  !IsDiagonal<T2>::value &&
193  !IsResizable<typename T1::ElementType>::value &&
194  !IsResizable<ET2>::value };
195  };
197  //**********************************************************************************************
198 
199  //**********************************************************************************************
201 
204  template< typename T1, typename T2, typename T3 >
205  struct UseDefaultKernel {
206  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
207  !UseOptimizedKernel<T1,T2,T3>::value };
208  };
210  //**********************************************************************************************
211 
212  public:
213  //**Type definitions****************************************************************************
220  typedef const ElementType ReturnType;
221  typedef const ResultType CompositeType;
222 
224  typedef typename SelectType< IsExpression<MT1>::value, const MT1, const MT1& >::Type LeftOperand;
225 
227  typedef typename SelectType< IsExpression<MT2>::value, const MT2, const MT2& >::Type RightOperand;
228 
231 
234  //**********************************************************************************************
235 
236  //**Compilation flags***************************************************************************
238  enum { vectorizable = !IsDiagonal<MT1>::value &&
239  MT1::vectorizable &&
243 
245  enum { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
246  !evaluateRight && MT2::smpAssignable };
247  //**********************************************************************************************
248 
249  //**Constructor*********************************************************************************
255  explicit inline TDMatTSMatMultExpr( const MT1& lhs, const MT2& rhs )
256  : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
257  , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
258  {
259  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
260  }
261  //**********************************************************************************************
262 
263  //**Access operator*****************************************************************************
270  inline ReturnType operator()( size_t i, size_t j ) const {
271  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
272  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
273 
275 
276  ElementType tmp = ElementType();
277 
278  // Early exit
279  if( lhs_.columns() == 0UL )
280  return tmp;
281 
282  // Fast computation in case the right-hand side sparse matrix directly provides iterators
284  {
285  CT2 B( rhs_ ); // Evaluation of the right-hand side sparse matrix operand
286 
287  const ConstIterator end( ( IsLower<MT1>::value )
288  ?( IsStrictlyLower<MT1>::value ? B.lowerBound(i,j) : B.upperBound(i,j) )
289  :( B.end(j) ) );
290  ConstIterator element( ( IsUpper<MT1>::value )
291  ?( IsStrictlyUpper<MT1>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
292  :( B.begin(j) ) );
293 
294  if( element != end ) {
295  tmp = lhs_(i,element->index()) * element->value();
296  ++element;
297  for( ; element!=end; ++element ) {
298  tmp += lhs_(i,element->index()) * element->value();
299  }
300  }
301  }
302 
303  // Default computation in case the right-hand side sparse matrix doesn't provide iterators
304  else
305  {
306  const size_t kbegin( ( IsUpper<MT1>::value )
307  ?( ( IsLower<MT2>::value )
308  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
309  , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
310  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
311  :( ( IsLower<MT2>::value )
312  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
313  :( 0UL ) ) );
314  const size_t kend( ( IsLower<MT1>::value )
315  ?( ( IsUpper<MT2>::value )
316  ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
317  , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
318  :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
319  :( ( IsUpper<MT2>::value )
320  ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
321  :( lhs_.columns() ) ) );
322 
323  if( ( !IsTriangular<MT1>::value && !IsTriangular<MT2>::value ) || kbegin < kend ) {
324  tmp = lhs_(i,kbegin) * rhs_(kbegin,j);
325  for( size_t k=kbegin+1UL; k<kend; ++k ) {
326  tmp += lhs_(i,k) * rhs_(k,j);
327  }
328  }
329  }
330 
331  return tmp;
332  }
333  //**********************************************************************************************
334 
335  //**At function*********************************************************************************
343  inline ReturnType at( size_t i, size_t j ) const {
344  if( i >= lhs_.rows() ) {
345  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
346  }
347  if( j >= rhs_.columns() ) {
348  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
349  }
350  return (*this)(i,j);
351  }
352  //**********************************************************************************************
353 
354  //**Rows function*******************************************************************************
359  inline size_t rows() const {
360  return lhs_.rows();
361  }
362  //**********************************************************************************************
363 
364  //**Columns function****************************************************************************
369  inline size_t columns() const {
370  return rhs_.columns();
371  }
372  //**********************************************************************************************
373 
374  //**Left operand access*************************************************************************
379  inline LeftOperand leftOperand() const {
380  return lhs_;
381  }
382  //**********************************************************************************************
383 
384  //**Right operand access************************************************************************
389  inline RightOperand rightOperand() const {
390  return rhs_;
391  }
392  //**********************************************************************************************
393 
394  //**********************************************************************************************
400  template< typename T >
401  inline bool canAlias( const T* alias ) const {
402  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
403  }
404  //**********************************************************************************************
405 
406  //**********************************************************************************************
412  template< typename T >
413  inline bool isAliased( const T* alias ) const {
414  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
415  }
416  //**********************************************************************************************
417 
418  //**********************************************************************************************
423  inline bool isAligned() const {
424  return lhs_.isAligned();
425  }
426  //**********************************************************************************************
427 
428  //**********************************************************************************************
433  inline bool canSMPAssign() const {
434  return ( columns() > SMP_TDMATTSMATMULT_THRESHOLD );
435  }
436  //**********************************************************************************************
437 
438  private:
439  //**Member variables****************************************************************************
440  LeftOperand lhs_;
441  RightOperand rhs_;
442  //**********************************************************************************************
443 
444  //**Assignment to dense matrices****************************************************************
457  template< typename MT // Type of the target dense matrix
458  , bool SO > // Storage order of the target dense matrix
459  friend inline void assign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
460  {
462 
463  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
464  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
465 
466  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
467  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
468 
469  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
470  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
471  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
472  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
473  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
474  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
475 
476  TDMatTSMatMultExpr::selectAssignKernel( ~lhs, A, B );
477  }
479  //**********************************************************************************************
480 
481  //**Default assignment to dense matrices********************************************************
495  template< typename MT3 // Type of the left-hand side target matrix
496  , typename MT4 // Type of the left-hand side matrix operand
497  , typename MT5 > // Type of the right-hand side matrix operand
498  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
499  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
500  {
501  typedef typename MT5::ConstIterator ConstIterator;
502 
503  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
504 
505  reset( C );
506 
507  for( size_t ii=0UL; ii<A.rows(); ii+=block )
508  {
509  const size_t itmp( min( ii+block, A.rows() ) );
510 
511  for( size_t j=0UL; j<B.columns(); ++j )
512  {
513  ConstIterator element( B.begin(j) );
514  const ConstIterator end( B.end(j) );
515 
516  for( ; element!=end; ++element )
517  {
518  const size_t j1( element->index() );
519 
521  {
522  C(j1,j) = A(j1,j1) * element->value();
523  }
524  else
525  {
526  const size_t ibegin( ( IsLower<MT4>::value )
527  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
528  :( ii ) );
529  const size_t iend( ( IsUpper<MT4>::value )
530  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
531  :( itmp ) );
532 
533  if( IsTriangular<MT4>::value && ibegin >= iend )
534  continue;
535 
536  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
537 
538  for( size_t i=ibegin; i<iend; ++i ) {
539  if( isDefault( C(i,j) ) )
540  C(i,j) = A(i,j1) * element->value();
541  else
542  C(i,j) += A(i,j1) * element->value();
543  }
544  }
545  }
546  }
547  }
548  }
550  //**********************************************************************************************
551 
552  //**Optimized assignment to dense matrices******************************************************
566  template< typename MT3 // Type of the left-hand side target matrix
567  , typename MT4 // Type of the left-hand side matrix operand
568  , typename MT5 > // Type of the right-hand side matrix operand
569  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
570  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
571  {
572  typedef typename MT5::ConstIterator ConstIterator;
573 
574  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
575 
576  reset( C );
577 
578  for( size_t ii=0UL; ii<A.rows(); ii+=block )
579  {
580  const size_t itmp( min( ii+block, A.rows() ) );
581 
582  for( size_t j=0UL; j<B.columns(); ++j )
583  {
584  const ConstIterator end( B.end(j) );
585  ConstIterator element( B.begin(j) );
586 
587  const size_t nonzeros( B.nonZeros(j) );
588  const size_t kpos( nonzeros & size_t(-4) );
589  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
590 
591  for( size_t k=0UL; k<kpos; k+=4UL )
592  {
593  const size_t j1( element->index() );
594  const ET2 v1( element->value() );
595  ++element;
596  const size_t j2( element->index() );
597  const ET2 v2( element->value() );
598  ++element;
599  const size_t j3( element->index() );
600  const ET2 v3( element->value() );
601  ++element;
602  const size_t j4( element->index() );
603  const ET2 v4( element->value() );
604  ++element;
605 
606  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
607 
608  const size_t ibegin( ( IsLower<MT4>::value )
609  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
610  :( ii ) );
611  const size_t iend( ( IsUpper<MT4>::value )
612  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
613  :( itmp ) );
614 
615  if( IsTriangular<MT4>::value && ibegin >= iend )
616  continue;
617 
618  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
619 
620  const size_t inum( iend - ibegin );
621  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
622  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
623 
624  for( size_t i=ibegin; i<ipos; i+=4UL ) {
625  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
626  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
627  C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
628  C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
629  }
630  for( size_t i=ipos; i<iend; ++i ) {
631  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
632  }
633  }
634 
635  for( ; element!=end; ++element )
636  {
637  const size_t j1( element->index() );
638  const ET2 v1( element->value() );
639 
640  const size_t ibegin( ( IsLower<MT4>::value )
641  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
642  :( ii ) );
643  const size_t iend( ( IsUpper<MT4>::value )
644  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
645  :( itmp ) );
646 
647  if( IsTriangular<MT4>::value && ibegin >= iend )
648  continue;
649 
650  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
651 
652  const size_t inum( iend - ibegin );
653  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
654  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
655 
656  for( size_t i=ibegin; i<ipos; i+=4UL ) {
657  C(i ,j) += A(i ,j1) * v1;
658  C(i+1UL,j) += A(i+1UL,j1) * v1;
659  C(i+2UL,j) += A(i+2UL,j1) * v1;
660  C(i+3UL,j) += A(i+3UL,j1) * v1;
661  }
662  for( size_t i=ipos; i<iend; ++i ) {
663  C(i,j) += A(i,j1) * v1;
664  }
665  }
666  }
667  }
668  }
670  //**********************************************************************************************
671 
672  //**Vectorized assignment to column-major dense matrices****************************************
686  template< typename MT3 // Type of the left-hand side target matrix
687  , typename MT4 // Type of the left-hand side matrix operand
688  , typename MT5 > // Type of the right-hand side matrix operand
689  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
690  selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
691  {
692  typedef IntrinsicTrait<ElementType> IT;
693  typedef typename MT5::ConstIterator ConstIterator;
694 
695  const bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
696 
697  reset( C );
698 
699  for( size_t j=0UL; j<B.columns(); ++j )
700  {
701  const ConstIterator end( B.end(j) );
702  ConstIterator element( B.begin(j) );
703 
704  const size_t nonzeros( B.nonZeros(j) );
705  const size_t kpos( nonzeros & size_t(-4) );
706  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
707 
708  for( size_t k=0UL; k<kpos; k+=4UL )
709  {
710  const size_t j1( element->index() );
711  const ET2 v1( element->value() );
712  ++element;
713  const size_t j2( element->index() );
714  const ET2 v2( element->value() );
715  ++element;
716  const size_t j3( element->index() );
717  const ET2 v3( element->value() );
718  ++element;
719  const size_t j4( element->index() );
720  const ET2 v4( element->value() );
721  ++element;
722 
723  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
724 
725  const IntrinsicType xmm1( set( v1 ) );
726  const IntrinsicType xmm2( set( v2 ) );
727  const IntrinsicType xmm3( set( v3 ) );
728  const IntrinsicType xmm4( set( v4 ) );
729 
730  const size_t ibegin( ( IsLower<MT4>::value )
731  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
732  :( 0UL ) );
733  const size_t iend( ( IsUpper<MT4>::value )
734  ?( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL )
735  :( A.rows() ) );
736  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
737 
738  const size_t ipos( remainder ? ( iend & size_t(-IT::size) ) : iend );
739  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (IT::size) ) ) == ipos, "Invalid end calculation" );
740 
741  size_t i( ibegin );
742 
743  for( ; i<ipos; i+=IT::size ) {
744  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
745  }
746  for( ; remainder && i<iend; ++i ) {
747  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
748  }
749  }
750 
751  for( ; element!=end; ++element )
752  {
753  const size_t j1( element->index() );
754  const ET2 v1( element->value() );
755 
756  const IntrinsicType xmm1( set( v1 ) );
757 
758  const size_t ibegin( ( IsLower<MT4>::value )
759  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
760  :( 0UL ) );
761  const size_t iend( ( IsUpper<MT4>::value )
762  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
763  :( A.rows() ) );
764  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
765 
766  const size_t ipos( remainder ? ( iend & size_t(-IT::size) ) : iend );
767  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (IT::size) ) ) == ipos, "Invalid end calculation" );
768 
769  size_t i( ibegin );
770 
771  for( ; i<ipos; i+=IT::size ) {
772  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 );
773  }
774  for( ; remainder && i<iend; ++i ) {
775  C(i,j) += A(i,j1) * v1;
776  }
777  }
778  }
779  }
781  //**********************************************************************************************
782 
783  //**Assignment to sparse matrices***************************************************************
796  template< typename MT // Type of the target sparse matrix
797  , bool SO > // Storage order of the target sparse matrix
798  friend inline void assign( SparseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
799  {
801 
802  typedef typename SelectType< SO, ResultType, OppositeType >::Type TmpType;
803 
810 
811  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
812  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
813 
814  const TmpType tmp( serial( rhs ) );
815  assign( ~lhs, tmp );
816  }
818  //**********************************************************************************************
819 
820  //**Addition assignment to dense matrices*******************************************************
833  template< typename MT // Type of the target dense matrix
834  , bool SO > // Storage order of the target dense matrix
835  friend inline void addAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
836  {
838 
839  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
840  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
841 
842  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
843  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
844 
845  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
846  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
847  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
848  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
849  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
850  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
851 
852  TDMatTSMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
853  }
855  //**********************************************************************************************
856 
857  //**Default addition assignment to dense matrices***********************************************
871  template< typename MT3 // Type of the left-hand side target matrix
872  , typename MT4 // Type of the left-hand side matrix operand
873  , typename MT5 > // Type of the right-hand side matrix operand
874  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
875  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
876  {
877  typedef typename MT5::ConstIterator ConstIterator;
878 
879  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
880 
881  for( size_t ii=0UL; ii<A.rows(); ii+=block )
882  {
883  const size_t itmp( min( ii+block, A.rows() ) );
884 
885  for( size_t j=0UL; j<B.columns(); ++j )
886  {
887  ConstIterator element( B.begin(j) );
888  const ConstIterator end( B.end(j) );
889 
890  for( ; element!=end; ++element )
891  {
892  const size_t j1( element->index() );
893 
894  if( IsDiagonal<MT4>::value )
895  {
896  C(j1,j) += A(j1,j1) * element->value();
897  }
898  else
899  {
900  const size_t ibegin( ( IsLower<MT4>::value )
901  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
902  :( ii ) );
903  const size_t iend( ( IsUpper<MT4>::value )
904  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
905  :( itmp ) );
906 
907  if( IsTriangular<MT4>::value && ibegin >= iend )
908  continue;
909 
910  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
911 
912  const size_t inum( iend - ibegin );
913  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
914  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
915 
916  for( size_t i=ibegin; i<ipos; i+=4UL ) {
917  C(i ,j) += A(i ,j1) * element->value();
918  C(i+1UL,j) += A(i+1UL,j1) * element->value();
919  C(i+2UL,j) += A(i+2UL,j1) * element->value();
920  C(i+3UL,j) += A(i+3UL,j1) * element->value();
921  }
922  for( size_t i=ipos; i<iend; ++i ) {
923  C(i,j) += A(i,j1) * element->value();
924  }
925  }
926  }
927  }
928  }
929  }
931  //**********************************************************************************************
932 
933  //**Optimized addition assignment to dense matrices*********************************************
947  template< typename MT3 // Type of the left-hand side target matrix
948  , typename MT4 // Type of the left-hand side matrix operand
949  , typename MT5 > // Type of the right-hand side matrix operand
950  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
951  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
952  {
953  typedef typename MT5::ConstIterator ConstIterator;
954 
955  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
956 
957  for( size_t ii=0UL; ii<A.rows(); ii+=block )
958  {
959  const size_t itmp( min( ii+block, A.rows() ) );
960 
961  for( size_t j=0UL; j<B.columns(); ++j )
962  {
963  const ConstIterator end( B.end(j) );
964  ConstIterator element( B.begin(j) );
965 
966  const size_t nonzeros( B.nonZeros(j) );
967  const size_t kpos( nonzeros & size_t(-4) );
968  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
969 
970  for( size_t k=0UL; k<kpos; k+=4UL )
971  {
972  const size_t j1( element->index() );
973  const ET2 v1( element->value() );
974  ++element;
975  const size_t j2( element->index() );
976  const ET2 v2( element->value() );
977  ++element;
978  const size_t j3( element->index() );
979  const ET2 v3( element->value() );
980  ++element;
981  const size_t j4( element->index() );
982  const ET2 v4( element->value() );
983  ++element;
984 
985  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
986 
987  const size_t ibegin( ( IsLower<MT4>::value )
988  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
989  :( ii ) );
990  const size_t iend( ( IsUpper<MT4>::value )
991  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
992  :( itmp ) );
993 
994  if( IsTriangular<MT4>::value && ibegin >= iend )
995  continue;
996 
997  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
998 
999  const size_t inum( iend - ibegin );
1000  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1001  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1002 
1003  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1004  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1005  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1006  C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1007  C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1008  }
1009  for( size_t i=ipos; i<iend; ++i ) {
1010  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1011  }
1012  }
1013 
1014  for( ; element!=end; ++element )
1015  {
1016  const size_t j1( element->index() );
1017  const ET2 v1( element->value() );
1018 
1019  const size_t ibegin( ( IsLower<MT4>::value )
1020  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1021  :( ii ) );
1022  const size_t iend( ( IsUpper<MT4>::value )
1023  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1024  :( itmp ) );
1025 
1026  if( IsTriangular<MT4>::value && ibegin >= iend )
1027  continue;
1028 
1029  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1030 
1031  const size_t inum( iend - ibegin );
1032  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1033  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1034 
1035  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1036  C(i ,j) += A(i ,j1) * v1;
1037  C(i+1UL,j) += A(i+1UL,j1) * v1;
1038  C(i+2UL,j) += A(i+2UL,j1) * v1;
1039  C(i+3UL,j) += A(i+3UL,j1) * v1;
1040  }
1041  for( size_t i=ipos; i<iend; ++i ) {
1042  C(i,j) += A(i,j1) * v1;
1043  }
1044  }
1045  }
1046  }
1047  }
1049  //**********************************************************************************************
1050 
1051  //**Vectorized addition assignment to column-major dense matrices*******************************
1065  template< typename MT3 // Type of the left-hand side target matrix
1066  , typename MT4 // Type of the left-hand side matrix operand
1067  , typename MT5 > // Type of the right-hand side matrix operand
1068  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
1069  selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1070  {
1071  typedef IntrinsicTrait<ElementType> IT;
1072  typedef typename MT5::ConstIterator ConstIterator;
1073 
1074  const bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
1075 
1076  for( size_t j=0UL; j<B.columns(); ++j )
1077  {
1078  const ConstIterator end( B.end(j) );
1079  ConstIterator element( B.begin(j) );
1080 
1081  const size_t nonzeros( B.nonZeros(j) );
1082  const size_t kpos( nonzeros & size_t(-4) );
1083  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1084 
1085  for( size_t k=0UL; k<kpos; k+=4UL )
1086  {
1087  const size_t j1( element->index() );
1088  const ET2 v1( element->value() );
1089  ++element;
1090  const size_t j2( element->index() );
1091  const ET2 v2( element->value() );
1092  ++element;
1093  const size_t j3( element->index() );
1094  const ET2 v3( element->value() );
1095  ++element;
1096  const size_t j4( element->index() );
1097  const ET2 v4( element->value() );
1098  ++element;
1099 
1100  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1101 
1102  const IntrinsicType xmm1( set( v1 ) );
1103  const IntrinsicType xmm2( set( v2 ) );
1104  const IntrinsicType xmm3( set( v3 ) );
1105  const IntrinsicType xmm4( set( v4 ) );
1106 
1107  const size_t ibegin( ( IsLower<MT4>::value )
1108  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
1109  :( 0UL ) );
1110  const size_t iend( ( IsUpper<MT4>::value )
1111  ?( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL )
1112  :( A.rows() ) );
1113  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1114 
1115  const size_t ipos( remainder ? ( iend & size_t(-IT::size) ) : iend );
1116  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (IT::size) ) ) == ipos, "Invalid end calculation" );
1117 
1118  size_t i( ibegin );
1119 
1120  for( ; i<ipos; i+=IT::size ) {
1121  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
1122  }
1123  for( ; remainder && i<iend; ++i ) {
1124  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1125  }
1126  }
1127 
1128  for( ; element!=end; ++element )
1129  {
1130  const size_t j1( element->index() );
1131  const ET2 v1( element->value() );
1132 
1133  const IntrinsicType xmm1( set( v1 ) );
1134 
1135  const size_t ibegin( ( IsLower<MT4>::value )
1136  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
1137  :( 0UL ) );
1138  const size_t iend( ( IsUpper<MT4>::value )
1139  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
1140  :( A.rows() ) );
1141  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1142 
1143  const size_t ipos( remainder ? ( iend & size_t(-IT::size) ) : iend );
1144  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (IT::size) ) ) == ipos, "Invalid end calculation" );
1145 
1146  size_t i( ibegin );
1147 
1148  for( ; i<ipos; i+=IT::size ) {
1149  C.store( i, j, C.load(i,j) + A.load(i,j1) * xmm1 );
1150  }
1151  for( ; remainder && i<iend; ++i ) {
1152  C(i,j) += A(i,j1) * v1;
1153  }
1154  }
1155  }
1156  }
1158  //**********************************************************************************************
1159 
1160  //**Addition assignment to sparse matrices******************************************************
1161  // No special implementation for the addition assignment to sparse matrices.
1162  //**********************************************************************************************
1163 
1164  //**Subtraction assignment to dense matrices****************************************************
1177  template< typename MT // Type of the target dense matrix
1178  , bool SO > // Storage order of the target dense matrix
1179  friend inline void subAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1180  {
1182 
1183  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1184  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1185 
1186  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1187  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1188 
1189  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1190  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1191  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1192  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1193  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1194  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1195 
1196  TDMatTSMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1197  }
1199  //**********************************************************************************************
1200 
1201  //**Default subtraction assignment to dense matrices********************************************
1215  template< typename MT3 // Type of the left-hand side target matrix
1216  , typename MT4 // Type of the left-hand side matrix operand
1217  , typename MT5 > // Type of the right-hand side matrix operand
1218  static inline typename EnableIf< UseDefaultKernel<MT3,MT4,MT5> >::Type
1219  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1220  {
1221  typedef typename MT5::ConstIterator ConstIterator;
1222 
1223  const size_t block( Or< IsColumnMajorMatrix<MT3>, IsDiagonal<MT4> >::value ? A.rows() : 64UL );
1224 
1225  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1226  {
1227  const size_t itmp( min( ii+block, A.rows() ) );
1228 
1229  for( size_t j=0UL; j<B.columns(); ++j )
1230  {
1231  ConstIterator element( B.begin(j) );
1232  const ConstIterator end( B.end(j) );
1233 
1234  for( ; element!=end; ++element )
1235  {
1236  const size_t j1( element->index() );
1237 
1238  if( IsDiagonal<MT4>::value )
1239  {
1240  C(j1,j) -= A(j1,j1) * element->value();
1241  }
1242  else
1243  {
1244  const size_t ibegin( ( IsLower<MT4>::value )
1245  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1246  :( ii ) );
1247  const size_t iend( ( IsUpper<MT4>::value )
1248  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1249  :( itmp ) );
1250 
1251  if( IsTriangular<MT4>::value && ibegin >= iend )
1252  continue;
1253 
1254  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1255 
1256  const size_t inum( iend - ibegin );
1257  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1258  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1259 
1260  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1261  C(i ,j) -= A(i ,j1) * element->value();
1262  C(i+1UL,j) -= A(i+1UL,j1) * element->value();
1263  C(i+2UL,j) -= A(i+2UL,j1) * element->value();
1264  C(i+3UL,j) -= A(i+3UL,j1) * element->value();
1265  }
1266  for( size_t i=ipos; i<iend; ++i ) {
1267  C(i,j) -= A(i,j1) * element->value();
1268  }
1269  }
1270  }
1271  }
1272  }
1273  }
1275  //**********************************************************************************************
1276 
1277  //**Optimized subtraction assignment to dense matrices******************************************
1291  template< typename MT3 // Type of the left-hand side target matrix
1292  , typename MT4 // Type of the left-hand side matrix operand
1293  , typename MT5 > // Type of the right-hand side matrix operand
1294  static inline typename EnableIf< UseOptimizedKernel<MT3,MT4,MT5> >::Type
1295  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1296  {
1297  typedef typename MT5::ConstIterator ConstIterator;
1298 
1299  const size_t block( IsColumnMajorMatrix<MT3>::value ? A.rows() : 64UL );
1300 
1301  for( size_t ii=0UL; ii<A.rows(); ii+=block )
1302  {
1303  const size_t itmp( min( ii+block, A.rows() ) );
1304 
1305  for( size_t j=0UL; j<B.columns(); ++j )
1306  {
1307  const ConstIterator end( B.end(j) );
1308  ConstIterator element( B.begin(j) );
1309 
1310  const size_t nonzeros( B.nonZeros(j) );
1311  const size_t kpos( nonzeros & size_t(-4) );
1312  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1313 
1314  for( size_t k=0UL; k<kpos; k+=4UL )
1315  {
1316  const size_t j1( element->index() );
1317  const ET2 v1( element->value() );
1318  ++element;
1319  const size_t j2( element->index() );
1320  const ET2 v2( element->value() );
1321  ++element;
1322  const size_t j3( element->index() );
1323  const ET2 v3( element->value() );
1324  ++element;
1325  const size_t j4( element->index() );
1326  const ET2 v4( element->value() );
1327  ++element;
1328 
1329  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1330 
1331  const size_t ibegin( ( IsLower<MT4>::value )
1332  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1333  :( ii ) );
1334  const size_t iend( ( IsUpper<MT4>::value )
1335  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL ) ) )
1336  :( itmp ) );
1337 
1338  if( IsTriangular<MT4>::value && ibegin >= iend )
1339  continue;
1340 
1341  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1342 
1343  const size_t inum( iend - ibegin );
1344  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1345  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1346 
1347  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1348  C(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1349  C(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1350  C(i+2UL,j) -= A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1351  C(i+3UL,j) -= A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1352  }
1353  for( size_t i=ipos; i<iend; ++i ) {
1354  C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1355  }
1356  }
1357 
1358  for( ; element!=end; ++element )
1359  {
1360  const size_t j1( element->index() );
1361  const ET2 v1( element->value() );
1362 
1363  const size_t ibegin( ( IsLower<MT4>::value )
1364  ?( max( ii, ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) ) )
1365  :( ii ) );
1366  const size_t iend( ( IsUpper<MT4>::value )
1367  ?( min( itmp, ( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL ) ) )
1368  :( itmp ) );
1369 
1370  if( IsTriangular<MT4>::value && ibegin >= iend )
1371  continue;
1372 
1373  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1374 
1375  const size_t inum( iend - ibegin );
1376  const size_t ipos( ibegin + ( inum & size_t(-4) ) );
1377  BLAZE_INTERNAL_ASSERT( ( ibegin + inum - ( inum % 4UL ) ) == ipos, "Invalid end calculation" );
1378 
1379  for( size_t i=ibegin; i<ipos; i+=4UL ) {
1380  C(i ,j) -= A(i ,j1) * v1;
1381  C(i+1UL,j) -= A(i+1UL,j1) * v1;
1382  C(i+2UL,j) -= A(i+2UL,j1) * v1;
1383  C(i+3UL,j) -= A(i+3UL,j1) * v1;
1384  }
1385  for( size_t i=ipos; i<iend; ++i ) {
1386  C(i,j) -= A(i,j1) * v1;
1387  }
1388  }
1389  }
1390  }
1391  }
1393  //**********************************************************************************************
1394 
1395  //**Vectorized subtraction assignment to column-major dense matrices****************************
1409  template< typename MT3 // Type of the left-hand side target matrix
1410  , typename MT4 // Type of the left-hand side matrix operand
1411  , typename MT5 > // Type of the right-hand side matrix operand
1412  static inline typename EnableIf< UseVectorizedKernel<MT3,MT4,MT5> >::Type
1413  selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1414  {
1415  typedef IntrinsicTrait<ElementType> IT;
1416  typedef typename MT5::ConstIterator ConstIterator;
1417 
1418  const bool remainder( !IsPadded<MT3>::value || !IsPadded<MT4>::value );
1419 
1420  for( size_t j=0UL; j<B.columns(); ++j )
1421  {
1422  const ConstIterator end( B.end(j) );
1423  ConstIterator element( B.begin(j) );
1424 
1425  const size_t nonzeros( B.nonZeros(j) );
1426  const size_t kpos( nonzeros & size_t(-4) );
1427  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1428 
1429  for( size_t k=0UL; k<kpos; k+=4UL )
1430  {
1431  const size_t j1( element->index() );
1432  const ET2 v1( element->value() );
1433  ++element;
1434  const size_t j2( element->index() );
1435  const ET2 v2( element->value() );
1436  ++element;
1437  const size_t j3( element->index() );
1438  const ET2 v3( element->value() );
1439  ++element;
1440  const size_t j4( element->index() );
1441  const ET2 v4( element->value() );
1442  ++element;
1443 
1444  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1445 
1446  const IntrinsicType xmm1( set( v1 ) );
1447  const IntrinsicType xmm2( set( v2 ) );
1448  const IntrinsicType xmm3( set( v3 ) );
1449  const IntrinsicType xmm4( set( v4 ) );
1450 
1451  const size_t ibegin( ( IsLower<MT4>::value )
1452  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
1453  :( 0UL ) );
1454  const size_t iend( ( IsUpper<MT4>::value )
1455  ?( IsStrictlyUpper<MT4>::value ? j4 : j4+1UL )
1456  :( A.rows() ) );
1457  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1458 
1459  const size_t ipos( remainder ? ( iend & size_t(-IT::size) ) : iend );
1460  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (IT::size) ) ) == ipos, "Invalid end calculation" );
1461 
1462  size_t i( ibegin );
1463 
1464  for( ; i<ipos; i+=IT::size ) {
1465  C.store( i, j, C.load(i,j) - A.load(i,j1) * xmm1 - A.load(i,j2) * xmm2 - A.load(i,j3) * xmm3 - A.load(i,j4) * xmm4 );
1466  }
1467  for( ; remainder && i<iend; ++i ) {
1468  C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1469  }
1470  }
1471 
1472  for( ; element!=end; ++element )
1473  {
1474  const size_t j1( element->index() );
1475  const ET2 v1( element->value() );
1476 
1477  const IntrinsicType xmm1( set( v1 ) );
1478 
1479  const size_t ibegin( ( IsLower<MT4>::value )
1480  ?( ( IsStrictlyLower<MT4>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
1481  :( 0UL ) );
1482  const size_t iend( ( IsUpper<MT4>::value )
1483  ?( IsStrictlyUpper<MT4>::value ? j1 : j1+1UL )
1484  :( A.rows() ) );
1485  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1486 
1487  const size_t ipos( remainder ? ( iend & size_t(-IT::size) ) : iend );
1488  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % (IT::size) ) ) == ipos, "Invalid end calculation" );
1489 
1490  size_t i( ibegin );
1491 
1492  for( ; i<ipos; i+=IT::size ) {
1493  C.store( i, j, C.load(i,j) - A.load(i,j1) * xmm1 );
1494  }
1495  for( ; remainder && i<iend; ++i ) {
1496  C(i,j) -= A(i,j1) * v1;
1497  }
1498  }
1499  }
1500  }
1502  //**********************************************************************************************
1503 
1504  //**Subtraction assignment to sparse matrices***************************************************
1505  // No special implementation for the subtraction assignment to sparse matrices.
1506  //**********************************************************************************************
1507 
1508  //**Multiplication assignment to dense matrices*************************************************
1509  // No special implementation for the multiplication assignment to dense matrices.
1510  //**********************************************************************************************
1511 
1512  //**Multiplication assignment to sparse matrices************************************************
1513  // No special implementation for the multiplication assignment to sparse matrices.
1514  //**********************************************************************************************
1515 
1516  //**SMP assignment to dense matrices************************************************************
1531  template< typename MT // Type of the target dense matrix
1532  , bool SO > // Storage order of the target dense matrix
1533  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1534  smpAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1535  {
1537 
1538  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1539  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1540 
1541  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1542  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1543 
1544  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1545  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1546  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1547  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1548  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1549  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1550 
1551  smpAssign( ~lhs, A * B );
1552  }
1554  //**********************************************************************************************
1555 
1556  //**SMP assignment to sparse matrices***********************************************************
1571  template< typename MT // Type of the target sparse matrix
1572  , bool SO > // Storage order of the target sparse matrix
1573  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1574  smpAssign( SparseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1575  {
1577 
1578  typedef typename SelectType< SO, ResultType, OppositeType >::Type TmpType;
1579 
1586 
1587  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1588  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1589 
1590  const TmpType tmp( rhs );
1591  smpAssign( ~lhs, tmp );
1592  }
1594  //**********************************************************************************************
1595 
1596  //**SMP addition assignment to dense matrices***************************************************
1611  template< typename MT // Type of the target dense matrix
1612  , bool SO > // Storage order of the target dense matrix
1613  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1614  smpAddAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1615  {
1617 
1618  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1619  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1620 
1621  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1622  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1623 
1624  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1625  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1626  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1627  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1628  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1629  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1630 
1631  smpAddAssign( ~lhs, A * B );
1632  }
1634  //**********************************************************************************************
1635 
1636  //**SMP addition assignment to sparse matrices**************************************************
1637  // No special implementation for the SMP addition assignment to sparse matrices.
1638  //**********************************************************************************************
1639 
1640  //**SMP subtraction assignment to dense matrices************************************************
1655  template< typename MT // Type of the target dense matrix
1656  , bool SO > // Storage order of the target dense matrix
1657  friend inline typename EnableIf< IsEvaluationRequired<MT,MT1,MT2> >::Type
1658  smpSubAssign( DenseMatrix<MT,SO>& lhs, const TDMatTSMatMultExpr& rhs )
1659  {
1661 
1662  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1663  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1664 
1665  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1666  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1667 
1668  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1669  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1670  BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1671  BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1672  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1673  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns() , "Invalid number of columns" );
1674 
1675  smpSubAssign( ~lhs, A * B );
1676  }
1678  //**********************************************************************************************
1679 
1680  //**SMP subtraction assignment to sparse matrices***********************************************
1681  // No special implementation for the SMP subtraction assignment to sparse matrices.
1682  //**********************************************************************************************
1683 
1684  //**SMP multiplication assignment to dense matrices*********************************************
1685  // No special implementation for the SMP multiplication assignment to dense matrices.
1686  //**********************************************************************************************
1687 
1688  //**SMP multiplication assignment to sparse matrices********************************************
1689  // No special implementation for the SMP multiplication assignment to sparse matrices.
1690  //**********************************************************************************************
1691 
1692  //**Compile time checks*************************************************************************
1700  //**********************************************************************************************
1701 };
1702 //*************************************************************************************************
1703 
1704 
1705 
1706 
1707 //=================================================================================================
1708 //
1709 // GLOBAL BINARY ARITHMETIC OPERATORS
1710 //
1711 //=================================================================================================
1712 
1713 //*************************************************************************************************
1742 template< typename T1 // Type of the left-hand side dense matrix
1743  , typename T2 > // Type of the right-hand side sparse matrix
1744 inline const TDMatTSMatMultExpr<T1,T2>
1746 {
1748 
1749  if( (~lhs).columns() != (~rhs).rows() ) {
1750  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1751  }
1752 
1753  return TDMatTSMatMultExpr<T1,T2>( ~lhs, ~rhs );
1754 }
1755 //*************************************************************************************************
1756 
1757 
1758 
1759 
1760 //=================================================================================================
1761 //
1762 // ROWS SPECIALIZATIONS
1763 //
1764 //=================================================================================================
1765 
1766 //*************************************************************************************************
1768 template< typename MT1, typename MT2 >
1769 struct Rows< TDMatTSMatMultExpr<MT1,MT2> > : public Rows<MT1>
1770 {};
1772 //*************************************************************************************************
1773 
1774 
1775 
1776 
1777 //=================================================================================================
1778 //
1779 // COLUMNS SPECIALIZATIONS
1780 //
1781 //=================================================================================================
1782 
1783 //*************************************************************************************************
1785 template< typename MT1, typename MT2 >
1786 struct Columns< TDMatTSMatMultExpr<MT1,MT2> > : public Columns<MT2>
1787 {};
1789 //*************************************************************************************************
1790 
1791 
1792 
1793 
1794 //=================================================================================================
1795 //
1796 // ISALIGNED SPECIALIZATIONS
1797 //
1798 //=================================================================================================
1799 
1800 //*************************************************************************************************
1802 template< typename MT1, typename MT2 >
1803 struct IsAligned< TDMatTSMatMultExpr<MT1,MT2> > : public IsTrue< IsAligned<MT1>::value >
1804 {};
1806 //*************************************************************************************************
1807 
1808 
1809 
1810 
1811 //=================================================================================================
1812 //
1813 // ISLOWER SPECIALIZATIONS
1814 //
1815 //=================================================================================================
1816 
1817 //*************************************************************************************************
1819 template< typename MT1, typename MT2 >
1820 struct IsLower< TDMatTSMatMultExpr<MT1,MT2> >
1821  : public IsTrue< And< IsLower<MT1>, IsLower<MT2> >::value >
1822 {};
1824 //*************************************************************************************************
1825 
1826 
1827 
1828 
1829 //=================================================================================================
1830 //
1831 // ISUNILOWER SPECIALIZATIONS
1832 //
1833 //=================================================================================================
1834 
1835 //*************************************************************************************************
1837 template< typename MT1, typename MT2 >
1838 struct IsUniLower< TDMatTSMatMultExpr<MT1,MT2> >
1839  : public IsTrue< And< IsUniLower<MT1>, IsUniLower<MT2> >::value >
1840 {};
1842 //*************************************************************************************************
1843 
1844 
1845 
1846 
1847 //=================================================================================================
1848 //
1849 // ISSTRICTLYLOWER SPECIALIZATIONS
1850 //
1851 //=================================================================================================
1852 
1853 //*************************************************************************************************
1855 template< typename MT1, typename MT2 >
1856 struct IsStrictlyLower< TDMatTSMatMultExpr<MT1,MT2> >
1857  : public IsTrue< Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
1858  , And< IsStrictlyLower<MT2>, IsLower<MT1> > >::value >
1859 {};
1861 //*************************************************************************************************
1862 
1863 
1864 
1865 
1866 //=================================================================================================
1867 //
1868 // ISUPPER SPECIALIZATIONS
1869 //
1870 //=================================================================================================
1871 
1872 //*************************************************************************************************
1874 template< typename MT1, typename MT2 >
1875 struct IsUpper< TDMatTSMatMultExpr<MT1,MT2> >
1876  : public IsTrue< And< IsUpper<MT1>, IsUpper<MT2> >::value >
1877 {};
1879 //*************************************************************************************************
1880 
1881 
1882 
1883 
1884 //=================================================================================================
1885 //
1886 // ISUNIUPPER SPECIALIZATIONS
1887 //
1888 //=================================================================================================
1889 
1890 //*************************************************************************************************
1892 template< typename MT1, typename MT2 >
1893 struct IsUniUpper< TDMatTSMatMultExpr<MT1,MT2> >
1894  : public IsTrue< And< IsUniUpper<MT1>, IsUniUpper<MT2> >::value >
1895 {};
1897 //*************************************************************************************************
1898 
1899 
1900 
1901 
1902 //=================================================================================================
1903 //
1904 // ISSTRICTLYUPPER SPECIALIZATIONS
1905 //
1906 //=================================================================================================
1907 
1908 //*************************************************************************************************
1910 template< typename MT1, typename MT2 >
1911 struct IsStrictlyUpper< TDMatTSMatMultExpr<MT1,MT2> >
1912  : public IsTrue< Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
1913  , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > >::value >
1914 {};
1916 //*************************************************************************************************
1917 
1918 
1919 
1920 
1921 //=================================================================================================
1922 //
1923 // EXPRESSION TRAIT SPECIALIZATIONS
1924 //
1925 //=================================================================================================
1926 
1927 //*************************************************************************************************
1929 template< typename MT1, typename MT2, typename VT >
1930 struct TDMatDVecMultExprTrait< TDMatTSMatMultExpr<MT1,MT2>, VT >
1931 {
1932  public:
1933  //**********************************************************************************************
1934  typedef typename SelectType< IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
1935  IsSparseMatrix<MT2>::value && IsColumnMajorMatrix<MT2>::value &&
1936  IsDenseVector<VT>::value && IsColumnVector<VT>::value
1937  , typename TDMatDVecMultExprTrait< MT1, typename TSMatDVecMultExprTrait<MT2,VT>::Type >::Type
1938  , INVALID_TYPE >::Type Type;
1939  //**********************************************************************************************
1940 };
1942 //*************************************************************************************************
1943 
1944 
1945 //*************************************************************************************************
1947 template< typename MT1, typename MT2, typename VT >
1948 struct TDMatSVecMultExprTrait< TDMatTSMatMultExpr<MT1,MT2>, VT >
1949 {
1950  public:
1951  //**********************************************************************************************
1952  typedef typename SelectType< IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
1953  IsSparseMatrix<MT2>::value && IsColumnMajorMatrix<MT2>::value &&
1954  IsSparseVector<VT>::value && IsColumnVector<VT>::value
1955  , typename TDMatDVecMultExprTrait< MT1, typename TSMatDVecMultExprTrait<MT2,VT>::Type >::Type
1956  , INVALID_TYPE >::Type Type;
1957  //**********************************************************************************************
1958 };
1960 //*************************************************************************************************
1961 
1962 
1963 //*************************************************************************************************
1965 template< typename VT, typename MT1, typename MT2 >
1966 struct TDVecTDMatMultExprTrait< VT, TDMatTSMatMultExpr<MT1,MT2> >
1967 {
1968  public:
1969  //**********************************************************************************************
1970  typedef typename SelectType< IsDenseVector<VT>::value && IsRowVector<VT>::value &&
1971  IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
1972  IsSparseMatrix<MT2>::value && IsColumnMajorMatrix<MT2>::value
1973  , typename TDVecTSMatMultExprTrait< typename TDVecTDMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
1974  , INVALID_TYPE >::Type Type;
1975  //**********************************************************************************************
1976 };
1978 //*************************************************************************************************
1979 
1980 
1981 //*************************************************************************************************
1983 template< typename VT, typename MT1, typename MT2 >
1984 struct TSVecTDMatMultExprTrait< VT, TDMatTSMatMultExpr<MT1,MT2> >
1985 {
1986  public:
1987  //**********************************************************************************************
1988  typedef typename SelectType< IsSparseVector<VT>::value && IsRowVector<VT>::value &&
1989  IsDenseMatrix<MT1>::value && IsColumnMajorMatrix<MT1>::value &&
1990  IsSparseMatrix<MT2>::value && IsColumnMajorMatrix<MT2>::value
1991  , typename TDVecTSMatMultExprTrait< typename TSVecTDMatMultExprTrait<VT,MT1>::Type, MT2 >::Type
1992  , INVALID_TYPE >::Type Type;
1993  //**********************************************************************************************
1994 };
1996 //*************************************************************************************************
1997 
1998 
1999 //*************************************************************************************************
2001 template< typename MT1, typename MT2, bool AF >
2002 struct SubmatrixExprTrait< TDMatTSMatMultExpr<MT1,MT2>, AF >
2003 {
2004  public:
2005  //**********************************************************************************************
2006  typedef typename MultExprTrait< typename SubmatrixExprTrait<const MT1,AF>::Type
2007  , typename SubmatrixExprTrait<const MT2,AF>::Type >::Type Type;
2008  //**********************************************************************************************
2009 };
2011 //*************************************************************************************************
2012 
2013 
2014 //*************************************************************************************************
2016 template< typename MT1, typename MT2 >
2017 struct RowExprTrait< TDMatTSMatMultExpr<MT1,MT2> >
2018 {
2019  public:
2020  //**********************************************************************************************
2021  typedef typename MultExprTrait< typename RowExprTrait<const MT1>::Type, MT2 >::Type Type;
2022  //**********************************************************************************************
2023 };
2025 //*************************************************************************************************
2026 
2027 
2028 //*************************************************************************************************
2030 template< typename MT1, typename MT2 >
2031 struct ColumnExprTrait< TDMatTSMatMultExpr<MT1,MT2> >
2032 {
2033  public:
2034  //**********************************************************************************************
2035  typedef typename MultExprTrait< MT1, typename ColumnExprTrait<const MT2>::Type >::Type Type;
2036  //**********************************************************************************************
2037 };
2039 //*************************************************************************************************
2040 
2041 } // namespace blaze
2042 
2043 #endif
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exceptionThis macro encapsulates the default way of...
Definition: Exception.h:187
const MT::ElementType max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1729
BLAZE_ALWAYS_INLINE EnableIf< And< IsIntegral< T >, HasSize< T, 2UL > >, simd_int16_t >::Type set(T value)
Sets all values in the vector to the given 2-byte integral value.
Definition: Set.h:73
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:89
Header file for mathematical functions.
TDMatTSMatMultExpr< MT1, MT2 > This
Type of this TDMatTSMatMultExpr instance.
Definition: TDMatTSMatMultExpr.h:214
Header file for the Rows type trait.
Header file for the IsUniUpper type trait.
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:7820
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:105
Header file for basic type definitions.
BLAZE_ALWAYS_INLINE size_t size(const Vector< VT, TF > &vector)
Returns the current size/dimension of the vector.
Definition: Vector.h:252
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:250
RT2::ElementType ET2
Element type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:134
Expression object for transpose dense matrix-transpose sparse matrix multiplications.The TDMatTSMatMultExpr class represents the compile time expression for multiplications between a column-major dense matrix and a column-major sparse matrix.
Definition: Forward.h:145
Header file for the IsSparseMatrix type trait.
Efficient implementation of a compressed matrix.The CompressedMatrix class template is the represent...
Definition: CompressedMatrix.h:207
Header file for the IsDiagonal type trait.
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:79
Header file for the ColumnExprTrait class template.
Header file for the IsColumnMajorMatrix type trait.
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:507
MT2::CompositeType CT2
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:136
MultTrait< RT1, RT2 >::Type ResultType
Result type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:215
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:2588
Header file for the IsRowVector type trait.
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:259
Header file for the And class template.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:90
SelectType< IsExpression< MT2 >::value, const MT2, const MT2 & >::Type RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:227
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:220
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:721
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Type relationship analysis.This class tests if the two data types A and B are equal. For this type comparison, the cv-qualifiers of both data types are ignored. If A and B are the same data type (ignoring the cv-qualifiers), then the value member enumeration is set to 1, the nested type definition Type is TrueType, and the class derives from TrueType. Otherwise value is set to 0, Type is FalseType, and the class derives from FalseType.
Definition: IsSame.h:158
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:90
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Header file for the IsUniLower type trait.
SelectType< evaluateRight, const RT2, CT2 >::Type RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: TDMatTSMatMultExpr.h:233
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:117
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:547
Constraint on the data type.
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: TDMatTSMatMultExpr.h:343
size_t columns() const
Returns the current number of columns of the matrix.
Definition: TDMatTSMatMultExpr.h:369
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatTSMatMultExpr.h:221
Constraint on the data type.
Constraint on the data type.
Header file for the MultExprTrait class template.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:90
SelectType< evaluateLeft, const RT1, CT1 >::Type LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatTSMatMultExpr.h:230
IntrinsicTrait< ElementType >::Type IntrinsicType
Resulting intrinsic element type.
Definition: TDMatTSMatMultExpr.h:219
Compile time type selection.The SelectType class template selects one of the two given types T1 and T...
Definition: SelectType.h:59
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
ResultType::TransposeType TransposeType
Transpose type for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:217
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:79
Header file for the TSVecTDMatMultExprTrait class template.
TDMatTSMatMultExpr(const MT1 &lhs, const MT2 &rhs)
Constructor for the TDMatTSMatMultExpr class.
Definition: TDMatTSMatMultExpr.h:255
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2592
Header file for the Or class template.
Header file for the TDMatSVecMultExprTrait class template.
Header file for the TDVecTSMatMultExprTrait class template.
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exceptionThis macro encapsulates the default way of Bla...
Definition: Exception.h:331
const MT::ElementType min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1682
Header file for the DenseMatrix base class.
Header file for the Columns type trait.
Header file for the TSMatDVecMultExprTrait class template.
Header file for the IsLower type trait.
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:92
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:78
Header file for the IsTriangular type trait.
Constraints on the storage order of matrix types.
bool canAlias(const T *alias) const
Returns whether the expression can alias with the given address alias.
Definition: TDMatTSMatMultExpr.h:401
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:2586
Header file for the SelectType class template.
Header file for the RowExprTrait class template.
Header file for all forward declarations for expression class templates.
Header file for the IsDenseMatrix type trait.
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatTSMatMultExpr.h:440
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
Header file for the serial shim.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:165
LeftOperand leftOperand() const
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatTSMatMultExpr.h:379
ResultType::OppositeType OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: TDMatTSMatMultExpr.h:216
bool isAliased(const T *alias) const
Returns whether the expression is aliased with the given address alias.
Definition: TDMatTSMatMultExpr.h:413
EnableIf< IsDenseMatrix< MT1 > >::Type smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
Header file for the IsSparseVector type trait.
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:79
Removal of reference modifiers.The RemoveCV type trait removes any reference modifiers from the given...
Definition: RemoveReference.h:69
size_t rows() const
Returns the current number of rows of the matrix.
Definition: TDMatTSMatMultExpr.h:359
Intrinsic characteristics of data types.The IntrinsicTrait class template provides the intrinsic char...
Definition: IntrinsicTrait.h:1232
Header file for run time assertion macros.
Compile time check for column-major matrix types.This type trait tests whether or not the given templ...
Definition: IsColumnMajorMatrix.h:110
EnableIf< IsDenseMatrix< MT1 > >::Type smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Utility type for generic codes.
bool canSMPAssign() const
Returns whether the expression can be used in SMP assignments.
Definition: TDMatTSMatMultExpr.h:433
Base template for the MultTrait class.
Definition: MultTrait.h:138
bool isAligned() const
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatTSMatMultExpr.h:423
Substitution Failure Is Not An Error (SFINAE) class.The EnableIf class template is an auxiliary tool ...
Definition: EnableIf.h:184
const bool useOptimizedKernels
Configuration switch for optimized kernels.This configuration switch enables/disables all optimized c...
Definition: Optimizations.h:84
Header file for the reset shim.
MT1::ResultType RT1
Result type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:131
Header file for the isDefault shim.
MT1::CompositeType CT1
Composite type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:135
RT1::ElementType ET1
Element type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:133
Constraints on the storage order of matrix types.
Header file for the RemoveReference type trait.
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:122
Header file for the IsDenseVector type trait.
Header file for all intrinsic functionality.
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
ResultType::ElementType ElementType
Resulting element type.
Definition: TDMatTSMatMultExpr.h:218
Header file for the IsComputation type trait class.
MT2::ResultType RT2
Result type of the right-hand side sparse matrix expression.
Definition: TDMatTSMatMultExpr.h:132
Compile time logical or evaluation.The Or class template performs at compile time a logical or ('&&')...
Definition: Or.h:78
EnableIf< IsDenseMatrix< MT1 > >::Type smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
Header file for the TDMatDVecMultExprTrait class template.
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: TDMatTSMatMultExpr.h:270
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
This ResultType
Result type for expression template evaluations.
Definition: CompressedMatrix.h:2583
Header file for the IsTrue value trait.
RightOperand rightOperand() const
Returns the right-hand side transpose sparse matrix operand.
Definition: TDMatTSMatMultExpr.h:389
Header file for the IsUpper type trait.
Header file for exception macros.
Header file for the IsColumnVector type trait.
Constraint on the data type.
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: TDMatTSMatMultExpr.h:441
Header file for the IsResizable type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the TDVecTDMatMultExprTrait class template.
SelectType< IsExpression< MT1 >::value, const MT1, const MT1 & >::Type LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatTSMatMultExpr.h:224
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:79
Header file for the IsExpression type trait class.
Header file for the TSMatSVecMultExprTrait class template.
Header file for the FunctionTrace class.