TDMatSVecMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATSVECMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATSVECMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
54 #include <blaze/math/Intrinsics.h>
55 #include <blaze/math/shims/Reset.h>
76 #include <blaze/util/Assert.h>
78 #include <blaze/util/DisableIf.h>
79 #include <blaze/util/EnableIf.h>
80 #include <blaze/util/Exception.h>
82 #include <blaze/util/SelectType.h>
83 #include <blaze/util/Types.h>
86 
87 
88 namespace blaze {
89 
90 //=================================================================================================
91 //
92 // CLASS TDMATSVECMULTEXPR
93 //
94 //=================================================================================================
95 
96 //*************************************************************************************************
103 template< typename MT // Type of the left-hand side dense matrix
104  , typename VT > // Type of the right-hand side sparse vector
105 class TDMatSVecMultExpr : public DenseVector< TDMatSVecMultExpr<MT,VT>, false >
106  , private MatVecMultExpr
107  , private Computation
108 {
109  private:
110  //**Type definitions****************************************************************************
111  typedef typename MT::ResultType MRT;
112  typedef typename VT::ResultType VRT;
113  typedef typename MRT::ElementType MET;
114  typedef typename VRT::ElementType VET;
115  typedef typename MT::CompositeType MCT;
116  typedef typename VT::CompositeType VCT;
117  //**********************************************************************************************
118 
119  //**********************************************************************************************
121  enum { evaluateMatrix = ( IsComputation<MT>::value && IsSame<MET,VET>::value &&
123  //**********************************************************************************************
124 
125  //**********************************************************************************************
127  enum { evaluateVector = IsComputation<VT>::value || RequiresEvaluation<VT>::value };
128  //**********************************************************************************************
129 
130  //**********************************************************************************************
132 
136  template< typename T1 >
137  struct UseSMPAssign {
138  enum { value = ( evaluateMatrix || evaluateVector ) };
139  };
141  //**********************************************************************************************
142 
143  //**********************************************************************************************
145 
149  template< typename T1, typename T2, typename T3 >
150  struct UseVectorizedKernel {
151  enum { value = useOptimizedKernels &&
153  T1::vectorizable && T2::vectorizable &&
158  };
160  //**********************************************************************************************
161 
162  //**********************************************************************************************
164 
168  template< typename T1, typename T2, typename T3 >
169  struct UseOptimizedKernel {
170  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
174  };
176  //**********************************************************************************************
177 
178  //**********************************************************************************************
180 
183  template< typename T1, typename T2, typename T3 >
184  struct UseDefaultKernel {
185  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
186  !UseOptimizedKernel<T1,T2,T3>::value };
187  };
189  //**********************************************************************************************
190 
191  public:
192  //**Type definitions****************************************************************************
198  typedef const ElementType ReturnType;
199  typedef const ResultType CompositeType;
200 
202  typedef typename SelectType< IsExpression<MT>::value, const MT, const MT& >::Type LeftOperand;
203 
205  typedef typename SelectType< IsExpression<VT>::value, const VT, const VT& >::Type RightOperand;
206 
209 
212  //**********************************************************************************************
213 
214  //**Compilation flags***************************************************************************
216  enum { vectorizable = !IsDiagonal<MT>::value &&
217  MT::vectorizable &&
221 
223  enum { smpAssignable = !evaluateMatrix && MT::smpAssignable &&
224  !evaluateVector && VT::smpAssignable };
225  //**********************************************************************************************
226 
227  //**Constructor*********************************************************************************
233  explicit inline TDMatSVecMultExpr( const MT& mat, const VT& vec )
234  : mat_( mat ) // Left-hand side dense matrix of the multiplication expression
235  , vec_( vec ) // Right-hand side sparse vector of the multiplication expression
236  {
237  BLAZE_INTERNAL_ASSERT( mat_.columns() == vec_.size(), "Invalid matrix and vector sizes" );
238  }
239  //**********************************************************************************************
240 
241  //**Subscript operator**************************************************************************
247  inline ReturnType operator[]( size_t index ) const {
248  BLAZE_INTERNAL_ASSERT( index < mat_.rows(), "Invalid vector access index" );
249 
251 
252  VCT x( vec_ ); // Evaluation of the right-hand side sparse vector operand
253 
254  BLAZE_INTERNAL_ASSERT( x.size() == vec_.size(), "Invalid vector size" );
255 
256  const ConstIterator end( ( IsLower<MT>::value )
257  ?( IsStrictlyLower<MT>::value ? x.lowerBound( index ) : x.upperBound( index ) )
258  :( x.end() ) );
259  ConstIterator element( ( IsUpper<MT>::value )
260  ?( IsStrictlyUpper<MT>::value ? x.upperBound( index ) : x.lowerBound( index ) )
261  :( x.begin() ) );
262 
263  ElementType res = ElementType();
264 
265  if( element != end ) {
266  res = mat_( index, element->index() ) * element->value();
267  ++element;
268  for( ; element!=x.end(); ++element ) {
269  res += mat_( index, element->index() ) * element->value();
270  }
271  }
272 
273  return res;
274  }
275  //**********************************************************************************************
276 
277  //**At function*********************************************************************************
284  inline ReturnType at( size_t index ) const {
285  if( index >= mat_.rows() ) {
286  BLAZE_THROW_OUT_OF_RANGE( "Invalid vector access index" );
287  }
288  return (*this)[index];
289  }
290  //**********************************************************************************************
291 
292  //**Size function*******************************************************************************
297  inline size_t size() const {
298  return mat_.rows();
299  }
300  //**********************************************************************************************
301 
302  //**Left operand access*************************************************************************
307  inline LeftOperand leftOperand() const {
308  return mat_;
309  }
310  //**********************************************************************************************
311 
312  //**Right operand access************************************************************************
317  inline RightOperand rightOperand() const {
318  return vec_;
319  }
320  //**********************************************************************************************
321 
322  //**********************************************************************************************
328  template< typename T >
329  inline bool canAlias( const T* alias ) const {
330  return mat_.isAliased( alias ) || vec_.isAliased( alias );
331  }
332  //**********************************************************************************************
333 
334  //**********************************************************************************************
340  template< typename T >
341  inline bool isAliased( const T* alias ) const {
342  return mat_.isAliased( alias ) || vec_.isAliased( alias );
343  }
344  //**********************************************************************************************
345 
346  //**********************************************************************************************
351  inline bool isAligned() const {
352  return mat_.isAligned();
353  }
354  //**********************************************************************************************
355 
356  //**********************************************************************************************
361  inline bool canSMPAssign() const {
362  return ( size() > SMP_TDMATSVECMULT_THRESHOLD );
363  }
364  //**********************************************************************************************
365 
366  private:
367  //**Member variables****************************************************************************
368  LeftOperand mat_;
369  RightOperand vec_;
370  //**********************************************************************************************
371 
372  //**Assignment to dense vectors*****************************************************************
385  template< typename VT1 > // Type of the target dense vector
386  friend inline void assign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
387  {
389 
390  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
391 
392  // Evaluation of the right-hand side sparse vector operand
393  RT x( serial( rhs.vec_ ) );
394  if( x.nonZeros() == 0UL ) {
395  reset( ~lhs );
396  return;
397  }
398 
399  // Evaluation of the left-hand side dense matrix operand
400  LT A( serial( rhs.mat_ ) );
401 
402  // Checking the evaluated operands
403  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
404  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
405  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
406  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
407 
408  // Performing the dense matrix-sparse vector multiplication
409  TDMatSVecMultExpr::selectAssignKernel( ~lhs, A, x );
410  }
412  //**********************************************************************************************
413 
414  //**Default assignment to dense vectors*********************************************************
428  template< typename VT1 // Type of the left-hand side target vector
429  , typename MT1 // Type of the left-hand side matrix operand
430  , typename VT2 > // Type of the right-hand side vector operand
431  static inline typename EnableIf< UseDefaultKernel<VT1,MT1,VT2> >::Type
432  selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
433  {
435 
436  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
437 
438  const size_t M( A.rows() );
439 
440  ConstIterator element( x.begin() );
441  const ConstIterator end( x.end() );
442 
443  size_t last( 0UL );
444 
445  if( IsLower<MT1>::value ) {
446  const size_t iend( IsStrictlyLower<MT1>::value ? element->index()+1UL : element->index() );
447  for( size_t i=0UL; i<iend; ++i )
448  reset( y[i] );
449  }
450 
451  for( ; element!=end; ++element )
452  {
453  const size_t index( element->index() );
454 
455  if( IsDiagonal<MT1>::value )
456  {
457  for( size_t i=last; i<index; ++i )
458  reset( y[i] );
459 
460  y[index] = A(index,index) * element->value();
461  last = index + 1UL;
462  }
463  else
464  {
465  const size_t ibegin( ( IsLower<MT1>::value )
466  ?( IsStrictlyLower<MT1>::value ? index+1UL : index )
467  :( 0UL ) );
468  const size_t iend( ( IsUpper<MT1>::value )
469  ?( IsStrictlyUpper<MT1>::value ? index : index+1UL )
470  :( M ) );
471  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
472 
473  for( size_t i=ibegin; i<last; ++i ) {
474  y[i] += A(i,index) * element->value();
475  }
476  for( size_t i=last; i<iend; ++i ) {
477  y[i] = A(i,index) * element->value();
478  }
479 
480  last = iend;
481  }
482  }
483 
484  if( IsUpper<MT1>::value ) {
485  for( size_t i=last; i<M; ++i )
486  reset( y[i] );
487  }
488  }
490  //**********************************************************************************************
491 
492  //**Optimized assignment to dense vectors*******************************************************
506  template< typename VT1 // Type of the left-hand side target vector
507  , typename MT1 // Type of the left-hand side matrix operand
508  , typename VT2 > // Type of the right-hand side vector operand
509  static inline typename EnableIf< UseOptimizedKernel<VT1,MT1,VT2> >::Type
510  selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
511  {
513 
514  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
515 
516  const size_t M( A.rows() );
517 
518  ConstIterator element( x.begin() );
519  const ConstIterator end( x.end() );
520 
521  const size_t jpos( x.nonZeros() & size_t(-4) );
522  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
523 
524  if( jpos > 3UL )
525  {
526  const size_t j1( element->index() );
527  const VET v1( element->value() );
528  ++element;
529  const size_t j2( element->index() );
530  const VET v2( element->value() );
531  ++element;
532  const size_t j3( element->index() );
533  const VET v3( element->value() );
534  ++element;
535  const size_t j4( element->index() );
536  const VET v4( element->value() );
537  ++element;
538 
539  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
540 
541  for( size_t i=0UL; i<M; ++i ) {
542  y[i] = A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
543  }
544  }
545  else
546  {
547  const size_t j1( element->index() );
548  const VET v1( element->value() );
549  ++element;
550 
551  for( size_t i=0UL; i<M; ++i ) {
552  y[i] = A(i,j1) * v1;
553  }
554  }
555 
556  for( size_t j=(jpos>3UL)?(4UL):(1UL); (j+4UL)<=jpos; j+=4UL )
557  {
558  const size_t j1( element->index() );
559  const VET v1( element->value() );
560  ++element;
561  const size_t j2( element->index() );
562  const VET v2( element->value() );
563  ++element;
564  const size_t j3( element->index() );
565  const VET v3( element->value() );
566  ++element;
567  const size_t j4( element->index() );
568  const VET v4( element->value() );
569  ++element;
570 
571  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
572 
573  const size_t ibegin( ( IsLower<MT1>::value )
574  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
575  :( 0UL ) );
576  const size_t iend( ( IsUpper<MT1>::value )
577  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
578  :( M ) );
579  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
580 
581  for( size_t i=ibegin; i<iend; ++i ) {
582  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
583  }
584  }
585  for( ; element!=end; ++element )
586  {
587  const size_t j1( element->index() );
588  const VET v1( element->value() );
589 
590  const size_t ibegin( ( IsLower<MT1>::value )
591  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
592  :( 0UL ) );
593  const size_t iend( ( IsUpper<MT1>::value )
594  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
595  :( M ) );
596  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
597 
598  for( size_t i=ibegin; i<iend; ++i ) {
599  y[i] += A(i,j1) * v1;
600  }
601  }
602  }
604  //**********************************************************************************************
605 
606  //**Vectorized assignment to dense vectors******************************************************
620  template< typename VT1 // Type of the left-hand side target vector
621  , typename MT1 // Type of the left-hand side matrix operand
622  , typename VT2 > // Type of the right-hand side vector operand
623  static inline typename EnableIf< UseVectorizedKernel<VT1,MT1,VT2> >::Type
624  selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
625  {
626  typedef IntrinsicTrait<ElementType> IT;
628 
629  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
630 
631  const size_t M( A.rows() );
632 
633  const bool remainder( !IsPadded<MT1>::value || !IsPadded<VT1>::value );
634 
635  ConstIterator element( x.begin() );
636  const ConstIterator end( x.end() );
637 
638  const size_t jpos( x.nonZeros() & size_t(-4) );
639  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
640 
641  if( jpos > 3UL )
642  {
643  const size_t j1( element->index() );
644  const VET v1( element->value() );
645  ++element;
646  const size_t j2( element->index() );
647  const VET v2( element->value() );
648  ++element;
649  const size_t j3( element->index() );
650  const VET v3( element->value() );
651  ++element;
652  const size_t j4( element->index() );
653  const VET v4( element->value() );
654  ++element;
655 
656  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
657 
658  const IntrinsicType xmm1( set( v1 ) );
659  const IntrinsicType xmm2( set( v2 ) );
660  const IntrinsicType xmm3( set( v3 ) );
661  const IntrinsicType xmm4( set( v4 ) );
662 
663  const size_t ipos( remainder ? ( M & size_t(-IT::size) ) : M );
664  BLAZE_INTERNAL_ASSERT( !remainder || ( M - ( M % IT::size ) ) == ipos, "Invalid end calculation" );
665 
666  size_t i( 0UL );
667 
668  for( ; i<ipos; i+=IT::size ) {
669  y.store( i, A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
670  }
671  for( ; remainder && i<M; ++i ) {
672  y[i] = A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
673  }
674  }
675  else
676  {
677  const size_t j1( element->index() );
678  const VET v1( element->value() );
679  ++element;
680 
681  const IntrinsicType xmm1( set( v1 ) );
682 
683  const size_t ipos( remainder ? ( M & size_t(-IT::size) ) : M );
684  BLAZE_INTERNAL_ASSERT( !remainder || ( M - ( M % IT::size ) ) == ipos, "Invalid end calculation" );
685 
686  size_t i( 0UL );
687 
688  for( ; i<ipos; i+=IT::size ) {
689  y.store( i, A.load(i,j1) * xmm1 );
690  }
691  for( ; remainder && i<M; ++i ) {
692  y[i] = A(i,j1) * v1;
693  }
694  }
695 
696  for( size_t j=(jpos>3UL)?(4UL):(1UL); (j+4UL)<=jpos; j+=4UL )
697  {
698  const size_t j1( element->index() );
699  const VET v1( element->value() );
700  ++element;
701  const size_t j2( element->index() );
702  const VET v2( element->value() );
703  ++element;
704  const size_t j3( element->index() );
705  const VET v3( element->value() );
706  ++element;
707  const size_t j4( element->index() );
708  const VET v4( element->value() );
709  ++element;
710 
711  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
712 
713  const IntrinsicType xmm1( set( v1 ) );
714  const IntrinsicType xmm2( set( v2 ) );
715  const IntrinsicType xmm3( set( v3 ) );
716  const IntrinsicType xmm4( set( v4 ) );
717 
718  const size_t ibegin( ( IsLower<MT1>::value )
719  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
720  :( 0UL ) );
721  const size_t iend( ( IsUpper<MT1>::value )
722  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
723  :( M ) );
724  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
725 
726  const size_t ipos( remainder ? ( iend & size_t(-IT::size) ) : iend );
727  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % IT::size ) ) == ipos, "Invalid end calculation" );
728 
729  size_t i( ibegin );
730 
731  for( ; i<ipos; i+=IT::size ) {
732  y.store( i, y.load(i) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
733  }
734  for( ; remainder && i<iend; ++i ) {
735  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
736  }
737  }
738 
739  for( ; element!=end; ++element )
740  {
741  const size_t j1( element->index() );
742  const VET v1( element->value() );
743 
744  const IntrinsicType xmm1( set( v1 ) );
745 
746  const size_t ibegin( ( IsLower<MT1>::value )
747  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
748  :( 0UL ) );
749  const size_t iend( ( IsUpper<MT1>::value )
750  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
751  :( M ) );
752  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
753 
754  const size_t ipos( remainder ? ( iend & size_t(-IT::size) ) : iend );
755  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % IT::size ) ) == ipos, "Invalid end calculation" );
756 
757  size_t i( ibegin );
758 
759  for( ; i<ipos; i+=IT::size ) {
760  y.store( i, y.load(i) + A.load(i,j1) * xmm1 );
761  }
762  for( ; remainder && i<iend; ++i ) {
763  y[i] += A(i,j1) * v1;
764  }
765  }
766  }
768  //**********************************************************************************************
769 
770  //**Assignment to sparse vectors****************************************************************
783  template< typename VT1 > // Type of the target sparse vector
784  friend inline void assign( SparseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
785  {
787 
791 
792  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
793 
794  const ResultType tmp( serial( rhs ) );
795  assign( ~lhs, tmp );
796  }
798  //**********************************************************************************************
799 
800  //**Addition assignment to dense vectors********************************************************
813  template< typename VT1 > // Type of the target dense vector
814  friend inline void addAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
815  {
817 
818  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
819 
821 
822  // Evaluation of the right-hand side sparse vector operand
823  RT x( serial( rhs.vec_ ) );
824  if( x.nonZeros() == 0UL ) return;
825 
826  // Evaluation of the left-hand side dense matrix operand
827  LT A( serial( rhs.mat_ ) );
828 
829  // Checking the evaluated operands
830  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
831  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
832  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
833  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
834 
835  // Performing the dense matrix-sparse vector multiplication
836  TDMatSVecMultExpr::selectAddAssignKernel( ~lhs, A, x );
837  }
839  //**********************************************************************************************
840 
841  //**Default addition assignment to dense vectors************************************************
855  template< typename VT1 // Type of the left-hand side target vector
856  , typename MT1 // Type of the left-hand side matrix operand
857  , typename VT2 > // Type of the right-hand side vector operand
858  static inline typename EnableIf< UseDefaultKernel<VT1,MT1,VT2> >::Type
859  selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
860  {
862 
863  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
864 
865  const size_t M( A.rows() );
866 
867  ConstIterator element( x.begin() );
868  const ConstIterator end( x.end() );
869 
870  for( ; element!=end; ++element )
871  {
872  const size_t index( element->index() );
873 
874  if( IsDiagonal<MT1>::value )
875  {
876  y[index] += A(index,index) * element->value();
877  }
878  else
879  {
880  const size_t ibegin( ( IsLower<MT1>::value )
881  ?( IsStrictlyLower<MT1>::value ? index+1UL : index )
882  :( 0UL ) );
883  const size_t iend( ( IsUpper<MT1>::value )
884  ?( IsStrictlyUpper<MT1>::value ? index : index+1UL )
885  :( M ) );
886  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
887 
888  for( size_t i=ibegin; i<iend; ++i ) {
889  y[i] += A(i,index) * element->value();
890  }
891  }
892  }
893  }
895  //**********************************************************************************************
896 
897  //**Optimized addition assignment to dense vectors**********************************************
911  template< typename VT1 // Type of the left-hand side target vector
912  , typename MT1 // Type of the left-hand side matrix operand
913  , typename VT2 > // Type of the right-hand side vector operand
914  static inline typename EnableIf< UseOptimizedKernel<VT1,MT1,VT2> >::Type
915  selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
916  {
918 
919  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
920 
921  const size_t M( A.rows() );
922 
923  ConstIterator element( x.begin() );
924  const ConstIterator end( x.end() );
925 
926  const size_t jpos( x.nonZeros() & size_t(-4) );
927  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
928 
929  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
930  {
931  const size_t j1( element->index() );
932  const VET v1( element->value() );
933  ++element;
934  const size_t j2( element->index() );
935  const VET v2( element->value() );
936  ++element;
937  const size_t j3( element->index() );
938  const VET v3( element->value() );
939  ++element;
940  const size_t j4( element->index() );
941  const VET v4( element->value() );
942  ++element;
943 
944  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
945 
946  const size_t ibegin( ( IsLower<MT1>::value )
947  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
948  :( 0UL ) );
949  const size_t iend( ( IsUpper<MT1>::value )
950  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
951  :( M ) );
952  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
953 
954  for( size_t i=ibegin; i<iend; ++i ) {
955  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
956  }
957  }
958  for( ; element!=end; ++element )
959  {
960  const size_t j1( element->index() );
961  const VET v1( element->value() );
962 
963  const size_t ibegin( ( IsLower<MT1>::value )
964  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
965  :( 0UL ) );
966  const size_t iend( ( IsUpper<MT1>::value )
967  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
968  :( M ) );
969  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
970 
971  for( size_t i=ibegin; i<iend; ++i ) {
972  y[i] += A(i,j1) * v1;
973  }
974  }
975  }
977  //**********************************************************************************************
978 
979  //**Vectorized addition assignment to dense vectors*********************************************
993  template< typename VT1 // Type of the left-hand side target vector
994  , typename MT1 // Type of the left-hand side matrix operand
995  , typename VT2 > // Type of the right-hand side vector operand
996  static inline typename EnableIf< UseVectorizedKernel<VT1,MT1,VT2> >::Type
997  selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
998  {
999  typedef IntrinsicTrait<ElementType> IT;
1001 
1002  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1003 
1004  const size_t M( A.rows() );
1005 
1006  const bool remainder( !IsPadded<MT1>::value || !IsPadded<VT1>::value );
1007 
1008  ConstIterator element( x.begin() );
1009  const ConstIterator end( x.end() );
1010 
1011  const size_t jpos( x.nonZeros() & size_t(-4) );
1012  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1013 
1014  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1015  {
1016  const size_t j1( element->index() );
1017  const VET v1( element->value() );
1018  ++element;
1019  const size_t j2( element->index() );
1020  const VET v2( element->value() );
1021  ++element;
1022  const size_t j3( element->index() );
1023  const VET v3( element->value() );
1024  ++element;
1025  const size_t j4( element->index() );
1026  const VET v4( element->value() );
1027  ++element;
1028 
1029  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1030 
1031  const IntrinsicType xmm1( set( v1 ) );
1032  const IntrinsicType xmm2( set( v2 ) );
1033  const IntrinsicType xmm3( set( v3 ) );
1034  const IntrinsicType xmm4( set( v4 ) );
1035 
1036  const size_t ibegin( ( IsLower<MT1>::value )
1037  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
1038  :( 0UL ) );
1039  const size_t iend( ( IsUpper<MT1>::value )
1040  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
1041  :( M ) );
1042  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1043 
1044  const size_t ipos( remainder ? ( iend & size_t(-IT::size) ) : iend );
1045  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % IT::size ) ) == ipos, "Invalid end calculation" );
1046 
1047  size_t i( ibegin );
1048 
1049  for( ; i<ipos; i+=IT::size ) {
1050  y.store( i, y.load(i) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
1051  }
1052  for( ; remainder && i<iend; ++i ) {
1053  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1054  }
1055  }
1056  for( ; element!=end; ++element )
1057  {
1058  const size_t j1( element->index() );
1059  const VET v1( element->value() );
1060 
1061  const IntrinsicType xmm1( set( v1 ) );
1062 
1063  const size_t ibegin( ( IsLower<MT1>::value )
1064  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
1065  :( 0UL ) );
1066  const size_t iend( ( IsUpper<MT1>::value )
1067  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
1068  :( M ) );
1069  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1070 
1071  const size_t ipos( remainder ? ( iend & size_t(-IT::size) ) : iend );
1072  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % IT::size ) ) == ipos, "Invalid end calculation" );
1073 
1074  size_t i( ibegin );
1075 
1076  for( ; i<ipos; i+=IT::size ) {
1077  y.store( i, y.load(i) + A.load(i,j1) * xmm1 );
1078  }
1079  for( ; remainder && i<iend; ++i ) {
1080  y[i] += A(i,j1) * v1;
1081  }
1082  }
1083  }
1085  //**********************************************************************************************
1086 
1087  //**Addition assignment to sparse vectors*******************************************************
1088  // No special implementation for the addition assignment to sparse vectors.
1089  //**********************************************************************************************
1090 
1091  //**Subtraction assignment to dense vectors*****************************************************
1104  template< typename VT1 > // Type of the target dense vector
1105  friend inline void subAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1106  {
1108 
1109  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1110 
1112 
1113  // Evaluation of the right-hand side sparse vector operand
1114  RT x( serial( rhs.vec_ ) );
1115  if( x.nonZeros() == 0UL ) return;
1116 
1117  // Evaluation of the left-hand side dense matrix operand
1118  LT A( serial( rhs.mat_ ) );
1119 
1120  // Checking the evaluated operands
1121  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1122  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1123  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1124  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1125 
1126  // Performing the dense matrix-sparse vector multiplication
1127  TDMatSVecMultExpr::selectSubAssignKernel( ~lhs, A, x );
1128  }
1130  //**********************************************************************************************
1131 
1132  //**Default subtraction assignment to dense vectors*********************************************
1146  template< typename VT1 // Type of the left-hand side target vector
1147  , typename MT1 // Type of the left-hand side matrix operand
1148  , typename VT2 > // Type of the right-hand side vector operand
1149  static inline typename EnableIf< UseDefaultKernel<VT1,MT1,VT2> >::Type
1150  selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1151  {
1153 
1154  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1155 
1156  const size_t M( A.rows() );
1157 
1158  ConstIterator element( x.begin() );
1159  const ConstIterator end( x.end() );
1160 
1161  for( ; element!=end; ++element )
1162  {
1163  const size_t index( element->index() );
1164 
1165  if( IsDiagonal<MT1>::value )
1166  {
1167  y[index] -= A(index,index) * element->value();
1168  }
1169  else
1170  {
1171  const size_t ibegin( ( IsLower<MT1>::value )
1172  ?( IsStrictlyLower<MT1>::value ? index+1UL : index )
1173  :( 0UL ) );
1174  const size_t iend( ( IsUpper<MT1>::value )
1175  ?( IsStrictlyUpper<MT1>::value ? index : index+1UL )
1176  :( M ) );
1177  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1178 
1179  for( size_t i=ibegin; i<iend; ++i ) {
1180  y[i] -= A(i,index) * element->value();
1181  }
1182  }
1183  }
1184  }
1186  //**********************************************************************************************
1187 
1188  //**Optimized subtraction assignment to dense vectors*******************************************
1202  template< typename VT1 // Type of the left-hand side target vector
1203  , typename MT1 // Type of the left-hand side matrix operand
1204  , typename VT2 > // Type of the right-hand side vector operand
1205  static inline typename EnableIf< UseOptimizedKernel<VT1,MT1,VT2> >::Type
1206  selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1207  {
1209 
1210  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1211 
1212  const size_t M( A.rows() );
1213 
1214  ConstIterator element( x.begin() );
1215  const ConstIterator end( x.end() );
1216 
1217  const size_t jpos( x.nonZeros() & size_t(-4) );
1218  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1219 
1220  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1221  {
1222  const size_t j1( element->index() );
1223  const VET v1( element->value() );
1224  ++element;
1225  const size_t j2( element->index() );
1226  const VET v2( element->value() );
1227  ++element;
1228  const size_t j3( element->index() );
1229  const VET v3( element->value() );
1230  ++element;
1231  const size_t j4( element->index() );
1232  const VET v4( element->value() );
1233  ++element;
1234 
1235  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1236 
1237  const size_t ibegin( ( IsLower<MT1>::value )
1238  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
1239  :( 0UL ) );
1240  const size_t iend( ( IsUpper<MT1>::value )
1241  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
1242  :( M ) );
1243  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1244 
1245  for( size_t i=ibegin; i<iend; ++i ) {
1246  y[i] -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1247  }
1248  }
1249  for( ; element!=end; ++element )
1250  {
1251  const size_t j1( element->index() );
1252  const VET v1( element->value() );
1253 
1254  const size_t ibegin( ( IsLower<MT1>::value )
1255  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
1256  :( 0UL ) );
1257  const size_t iend( ( IsUpper<MT1>::value )
1258  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
1259  :( M ) );
1260  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1261 
1262  for( size_t i=ibegin; i<iend; ++i ) {
1263  y[i] -= A(i,j1) * v1;
1264  }
1265  }
1266  }
1268  //**********************************************************************************************
1269 
1270  //**Vectorized subtraction assignment to dense vectors******************************************
1284  template< typename VT1 // Type of the left-hand side target vector
1285  , typename MT1 // Type of the left-hand side matrix operand
1286  , typename VT2 > // Type of the right-hand side vector operand
1287  static inline typename EnableIf< UseVectorizedKernel<VT1,MT1,VT2> >::Type
1288  selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1289  {
1290  typedef IntrinsicTrait<ElementType> IT;
1292 
1293  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1294 
1295  const size_t M( A.rows() );
1296 
1297  const bool remainder( !IsPadded<MT1>::value || !IsPadded<VT1>::value );
1298 
1299  ConstIterator element( x.begin() );
1300  const ConstIterator end( x.end() );
1301 
1302  const size_t jpos( x.nonZeros() & size_t(-4) );
1303  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1304 
1305  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1306  {
1307  const size_t j1( element->index() );
1308  const VET v1( element->value() );
1309  ++element;
1310  const size_t j2( element->index() );
1311  const VET v2( element->value() );
1312  ++element;
1313  const size_t j3( element->index() );
1314  const VET v3( element->value() );
1315  ++element;
1316  const size_t j4( element->index() );
1317  const VET v4( element->value() );
1318  ++element;
1319 
1320  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1321 
1322  const IntrinsicType xmm1( set( v1 ) );
1323  const IntrinsicType xmm2( set( v2 ) );
1324  const IntrinsicType xmm3( set( v3 ) );
1325  const IntrinsicType xmm4( set( v4 ) );
1326 
1327  const size_t ibegin( ( IsLower<MT1>::value )
1328  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
1329  :( 0UL ) );
1330  const size_t iend( ( IsUpper<MT1>::value )
1331  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
1332  :( M ) );
1333  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1334 
1335  const size_t ipos( remainder ? ( iend & size_t(-IT::size) ) : iend );
1336  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % IT::size ) ) == ipos, "Invalid end calculation" );
1337 
1338  size_t i( ibegin );
1339 
1340  for( ; i<ipos; i+=IT::size ) {
1341  y.store( i, y.load(i) - A.load(i,j1) * xmm1 - A.load(i,j2) * xmm2 - A.load(i,j3) * xmm3 - A.load(i,j4) * xmm4 );
1342  }
1343  for( ; remainder && i<iend; ++i ) {
1344  y[i] -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1345  }
1346  }
1347  for( ; element!=end; ++element )
1348  {
1349  const size_t j1( element->index() );
1350  const VET v1( element->value() );
1351 
1352  const IntrinsicType xmm1( set( v1 ) );
1353 
1354  const size_t ibegin( ( IsLower<MT1>::value )
1355  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-IT::size) )
1356  :( 0UL ) );
1357  const size_t iend( ( IsUpper<MT1>::value )
1358  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
1359  :( M ) );
1360  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1361 
1362  const size_t ipos( remainder ? ( iend & size_t(-IT::size) ) : iend );
1363  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % IT::size ) ) == ipos, "Invalid end calculation" );
1364 
1365  size_t i( ibegin );
1366 
1367  for( ; i<ipos; i+=IT::size ) {
1368  y.store( i, y.load(i) - A.load(i,j1) * xmm1 );
1369  }
1370  for( ; remainder && i<iend; ++i ) {
1371  y[i] -= A(i,j1) * v1;
1372  }
1373  }
1374  }
1376  //**********************************************************************************************
1377 
1378  //**Subtraction assignment to sparse vectors****************************************************
1379  // No special implementation for the subtraction assignment to sparse vectors.
1380  //**********************************************************************************************
1381 
1382  //**Multiplication assignment to dense vectors**************************************************
1395  template< typename VT1 > // Type of the target dense vector
1396  friend inline void multAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1397  {
1399 
1403 
1404  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1405 
1406  const ResultType tmp( serial( rhs ) );
1407  multAssign( ~lhs, tmp );
1408  }
1410  //**********************************************************************************************
1411 
1412  //**Multiplication assignment to sparse vectors*************************************************
1413  // No special implementation for the multiplication assignment to sparse vectors.
1414  //**********************************************************************************************
1415 
1416  //**SMP assignment to dense vectors*****************************************************************
1431  template< typename VT1 > // Type of the target dense vector
1432  friend inline typename EnableIf< UseSMPAssign<VT1> >::Type
1433  smpAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1434  {
1436 
1437  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1438 
1439  // Evaluation of the right-hand side sparse vector operand
1440  RT x( rhs.vec_ );
1441  if( x.nonZeros() == 0UL ) {
1442  reset( ~lhs );
1443  return;
1444  }
1445 
1446  // Evaluation of the left-hand side dense matrix operand
1447  LT A( rhs.mat_ );
1448 
1449  // Checking the evaluated operands
1450  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1451  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1452  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1453  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1454 
1455  // Performing the dense matrix-sparse vector multiplication
1456  smpAssign( ~lhs, A * x );
1457  }
1459  //**********************************************************************************************
1460 
1461  //**SMP assignment to sparse vectors************************************************************
1476  template< typename VT1 > // Type of the target sparse vector
1477  friend inline typename EnableIf< UseSMPAssign<VT1> >::Type
1478  smpAssign( SparseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1479  {
1481 
1485 
1486  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1487 
1488  const ResultType tmp( rhs );
1489  smpAssign( ~lhs, tmp );
1490  }
1492  //**********************************************************************************************
1493 
1494  //**SMP addition assignment to dense vectors****************************************************
1509  template< typename VT1 > // Type of the target dense vector
1510  friend inline typename EnableIf< UseSMPAssign<VT1> >::Type
1511  smpAddAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1512  {
1514 
1515  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1516 
1518 
1519  // Evaluation of the right-hand side sparse vector operand
1520  RT x( rhs.vec_ );
1521  if( x.nonZeros() == 0UL ) return;
1522 
1523  // Evaluation of the left-hand side dense matrix operand
1524  LT A( rhs.mat_ );
1525 
1526  // Checking the evaluated operands
1527  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1528  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1529  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1530  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1531 
1532  // Performing the dense matrix-sparse vector multiplication
1533  smpAddAssign( ~lhs, A * x );
1534  }
1536  //**********************************************************************************************
1537 
1538  //**SMP addition assignment to sparse vectors***************************************************
1539  // No special implementation for the SMP addition assignment to sparse vectors.
1540  //**********************************************************************************************
1541 
1542  //**SMP subtraction assignment to dense vectors*************************************************
1557  template< typename VT1 > // Type of the target dense vector
1558  friend inline typename EnableIf< UseSMPAssign<VT1> >::Type
1559  smpSubAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1560  {
1562 
1563  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1564 
1566 
1567  // Evaluation of the right-hand side sparse vector operand
1568  RT x( rhs.vec_ );
1569  if( x.nonZeros() == 0UL ) return;
1570 
1571  // Evaluation of the left-hand side dense matrix operand
1572  LT A( rhs.mat_ );
1573 
1574  // Checking the evaluated operands
1575  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1576  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1577  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1578  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1579 
1580  // Performing the dense matrix-sparse vector multiplication
1581  smpSubAssign( ~lhs, A * x );
1582  }
1584  //**********************************************************************************************
1585 
1586  //**SMP subtraction assignment to sparse vectors************************************************
1587  // No special implementation for the SMP subtraction assignment to sparse vectors.
1588  //**********************************************************************************************
1589 
1590  //**SMP multiplication assignment to dense vectors**********************************************
1605  template< typename VT1 > // Type of the target dense vector
1606  friend inline typename EnableIf< UseSMPAssign<VT1> >::Type
1607  smpMultAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1608  {
1610 
1614 
1615  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1616 
1617  const ResultType tmp( rhs );
1618  smpMultAssign( ~lhs, tmp );
1619  }
1621  //**********************************************************************************************
1622 
1623  //**SMP multiplication assignment to sparse vectors*********************************************
1624  // No special implementation for the SMP multiplication assignment to sparse vectors.
1625  //**********************************************************************************************
1626 
1627  //**Compile time checks*************************************************************************
1635  //**********************************************************************************************
1636 };
1637 //*************************************************************************************************
1638 
1639 
1640 
1641 
1642 //=================================================================================================
1643 //
1644 // GLOBAL BINARY ARITHMETIC OPERATORS
1645 //
1646 //=================================================================================================
1647 
1648 //*************************************************************************************************
1680 template< typename T1 // Type of the left-hand side dense matrix
1681  , typename T2 > // Type of the right-hand side sparse vector
1682 inline const typename DisableIf< IsMatMatMultExpr<T1>, TDMatSVecMultExpr<T1,T2> >::Type
1684 {
1686 
1687  if( (~mat).columns() != (~vec).size() ) {
1688  BLAZE_THROW_INVALID_ARGUMENT( "Matrix and vector sizes do not match" );
1689  }
1690 
1691  return TDMatSVecMultExpr<T1,T2>( ~mat, ~vec );
1692 }
1693 //*************************************************************************************************
1694 
1695 
1696 
1697 
1698 //=================================================================================================
1699 //
1700 // SIZE SPECIALIZATIONS
1701 //
1702 //=================================================================================================
1703 
1704 //*************************************************************************************************
1706 template< typename MT, typename VT >
1707 struct Size< TDMatSVecMultExpr<MT,VT> > : public Rows<MT>
1708 {};
1710 //*************************************************************************************************
1711 
1712 
1713 
1714 
1715 //=================================================================================================
1716 //
1717 // ISALIGNED SPECIALIZATIONS
1718 //
1719 //=================================================================================================
1720 
1721 //*************************************************************************************************
1723 template< typename MT, typename VT >
1724 struct IsAligned< TDMatSVecMultExpr<MT,VT> > : public IsTrue< IsAligned<MT>::value >
1725 {};
1727 //*************************************************************************************************
1728 
1729 
1730 
1731 
1732 //=================================================================================================
1733 //
1734 // EXPRESSION TRAIT SPECIALIZATIONS
1735 //
1736 //=================================================================================================
1737 
1738 //*************************************************************************************************
1740 template< typename MT, typename VT, bool AF >
1741 struct SubvectorExprTrait< TDMatSVecMultExpr<MT,VT>, AF >
1742 {
1743  public:
1744  //**********************************************************************************************
1745  typedef typename MultExprTrait< typename SubmatrixExprTrait<const MT,AF>::Type
1746  , typename SubvectorExprTrait<const VT,AF>::Type >::Type Type;
1747  //**********************************************************************************************
1748 };
1750 //*************************************************************************************************
1751 
1752 } // namespace blaze
1753 
1754 #endif
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exceptionThis macro encapsulates the default way of...
Definition: Exception.h:187
BLAZE_ALWAYS_INLINE EnableIf< And< IsIntegral< T >, HasSize< T, 2UL > >, simd_int16_t >::Type set(T value)
Sets all values in the vector to the given 2-byte integral value.
Definition: Set.h:73
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:89
RightOperand vec_
Right-hand side sparse vector of the multiplication expression.
Definition: TDMatSVecMultExpr.h:369
Header file for the Rows type trait.
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:7820
Header file for basic type definitions.
BLAZE_ALWAYS_INLINE size_t size(const Vector< VT, TF > &vector)
Returns the current size/dimension of the vector.
Definition: Vector.h:252
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:250
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a column dense or sparse vector type...
Definition: ColumnVector.h:79
Efficient implementation of a compressed matrix.The CompressedMatrix class template is the represent...
Definition: CompressedMatrix.h:207
Header file for the IsDiagonal type trait.
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:79
Header file for the IsSame and IsStrictlySame type traits.
IntrinsicTrait< ElementType >::Type IntrinsicType
Resulting intrinsic element type.
Definition: TDMatSVecMultExpr.h:197
VT::CompositeType VCT
Composite type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:116
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:507
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:2588
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:259
Header file for the DenseVector base class.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:90
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:721
Header file for the Computation base class.
Type relationship analysis.This class tests if the two data types A and B are equal. For this type comparison, the cv-qualifiers of both data types are ignored. If A and B are the same data type (ignoring the cv-qualifiers), then the value member enumeration is set to 1, the nested type definition Type is TrueType, and the class derives from TrueType. Otherwise value is set to 0, Type is FalseType, and the class derives from FalseType.
Definition: IsSame.h:158
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:198
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:90
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
SelectType< IsExpression< MT >::value, const MT, const MT & >::Type LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:202
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
Constraint on the data type.
ResultType::TransposeType TransposeType
Transpose type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:195
MT::ResultType MRT
Result type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:111
Constraint on the transpose flag of vector types.
Constraint on the data type.
Header file for the MultExprTrait class template.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:90
SelectType< evaluateVector, const VRT, typename VT::CompositeType >::Type RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: TDMatSVecMultExpr.h:211
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:261
Compile time type selection.The SelectType class template selects one of the two given types T1 and T...
Definition: SelectType.h:59
Header file for the DisableIf class template.
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
VRT::ElementType VET
Element type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:114
MultTrait< MRT, VRT >::Type ResultType
Result type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:194
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
MT::CompositeType MCT
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:115
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:79
LeftOperand leftOperand() const
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatSVecMultExpr.h:307
bool canSMPAssign() const
Returns whether the expression can be used in SMP assignments.
Definition: TDMatSVecMultExpr.h:361
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2592
ResultType::ElementType ElementType
Resulting element type.
Definition: TDMatSVecMultExpr.h:196
Header file for the IsMatMatMultExpr type trait class.
LeftOperand mat_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatSVecMultExpr.h:368
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exceptionThis macro encapsulates the default way of Bla...
Definition: Exception.h:331
MRT::ElementType MET
Element type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:113
Base class for N-dimensional dense vectors.The DenseVector class is a base class for all arbitrarily ...
Definition: DenseVector.h:70
Header file for the IsLower type trait.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional vector type...
Definition: SparseVector.h:79
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:92
SelectType< evaluateMatrix, const MRT, MCT >::Type LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatSVecMultExpr.h:208
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:78
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATVECMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/vector ...
Definition: MatVecMultExpr.h:166
Constraint on the data type.
SelectType< IsExpression< VT >::value, const VT, const VT & >::Type RightOperand
Composite type of the right-hand side dense vector expression.
Definition: TDMatSVecMultExpr.h:205
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:2586
Header file for the SelectType class template.
Header file for all forward declarations for expression class templates.
Constraint on the data type.
VT::ResultType VRT
Result type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:112
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
Header file for the serial shim.
bool canAlias(const T *alias) const
Returns whether the expression can alias with the given address alias.
Definition: TDMatSVecMultExpr.h:329
Expression object for transpose dense matrix-sparse vector multiplications.The TDMatSVecMultExpr clas...
Definition: Forward.h:143
Compile time check for resizable data types.This type trait tests whether the given data type is a re...
Definition: IsResizable.h:75
EnableIf< IsDenseMatrix< MT1 > >::Type smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
Header file for the SubmatrixExprTrait class template.
RightOperand rightOperand() const
Returns the right-hand side sparse vector operand.
Definition: TDMatSVecMultExpr.h:317
Removal of reference modifiers.The RemoveCV type trait removes any reference modifiers from the given...
Definition: RemoveReference.h:69
bool isAliased(const T *alias) const
Returns whether the expression is aliased with the given address alias.
Definition: TDMatSVecMultExpr.h:341
Intrinsic characteristics of data types.The IntrinsicTrait class template provides the intrinsic char...
Definition: IntrinsicTrait.h:1232
size_t size() const
Returns the current size/dimension of the vector.
Definition: TDMatSVecMultExpr.h:297
Header file for run time assertion macros.
EnableIf< IsDenseMatrix< MT1 > >::Type smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Base template for the MultTrait class.
Definition: MultTrait.h:138
Substitution Failure Is Not An Error (SFINAE) class.The EnableIf class template is an auxiliary tool ...
Definition: EnableIf.h:184
const bool useOptimizedKernels
Configuration switch for optimized kernels.This configuration switch enables/disables all optimized c...
Definition: Optimizations.h:84
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatSVecMultExpr.h:199
Header file for the reset shim.
Compile time check for data types.This type trait tests whether or not the given template parameter i...
Definition: IsBlasCompatible.h:99
Constraints on the storage order of matrix types.
Header file for the RemoveReference type trait.
Header file for all intrinsic functionality.
TDMatSVecMultExpr(const MT &mat, const VT &vec)
Constructor for the TDMatSVecMultExpr class.
Definition: TDMatSVecMultExpr.h:233
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional vector type...
Definition: DenseVector.h:79
ReturnType at(size_t index) const
Checked access to the vector elements.
Definition: TDMatSVecMultExpr.h:284
Header file for the IsComputation type trait class.
TDMatSVecMultExpr< MT, VT > This
Type of this TDMatSVecMultExpr instance.
Definition: TDMatSVecMultExpr.h:193
EnableIf< IsDenseMatrix< MT1 > >::Type smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
Base class for sparse vectors.The SparseVector class is a base class for all arbitrarily sized (N-dim...
Definition: Forward.h:118
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
This ResultType
Result type for expression template evaluations.
Definition: CompressedMatrix.h:2583
BLAZE_ALWAYS_INLINE size_t columns(const Matrix< MT, SO > &matrix)
Returns the current number of columns of the matrix.
Definition: Matrix.h:324
Header file for the SubvectorExprTrait class template.
Constraint on the data type.
Header file for the IsUpper type trait.
Header file for exception macros.
Header file for the MatVecMultExpr base class.
ReturnType operator[](size_t index) const
Subscript operator for the direct access to the vector elements.
Definition: TDMatSVecMultExpr.h:247
Header file for the IsResizable type trait.
EnableIf< IsDenseVector< VT1 > >::Type smpMultAssign(Vector< VT1, TF1 > &lhs, const Vector< VT2, TF2 > &rhs)
Default implementation of the SMP multiplication assignment of a vector to a dense vector...
Definition: DenseVector.h:189
Header file for the Size type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the IsExpression type trait class.
Header file for the FunctionTrace class.
bool isAligned() const
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatSVecMultExpr.h:351