TSVecDMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TSVECDMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TSVECDMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
54 #include <blaze/math/Intrinsics.h>
55 #include <blaze/math/shims/Reset.h>
76 #include <blaze/util/Assert.h>
78 #include <blaze/util/DisableIf.h>
79 #include <blaze/util/EnableIf.h>
80 #include <blaze/util/Exception.h>
82 #include <blaze/util/SelectType.h>
83 #include <blaze/util/Types.h>
86 
87 
88 namespace blaze {
89 
90 //=================================================================================================
91 //
92 // CLASS TSVECDMATMULTEXPR
93 //
94 //=================================================================================================
95 
96 //*************************************************************************************************
103 template< typename VT // Type of the left-hand side sparse vector
104  , typename MT > // Type of the right-hand side dense matrix
105 class TSVecDMatMultExpr : public DenseVector< TSVecDMatMultExpr<VT,MT>, true >
106  , private TVecMatMultExpr
107  , private Computation
108 {
109  private:
110  //**Type definitions****************************************************************************
111  typedef typename VT::ResultType VRT;
112  typedef typename MT::ResultType MRT;
113  typedef typename VRT::ElementType VET;
114  typedef typename MRT::ElementType MET;
115  typedef typename VT::CompositeType VCT;
116  typedef typename MT::CompositeType MCT;
117  //**********************************************************************************************
118 
119  //**********************************************************************************************
121  enum { evaluateVector = IsComputation<VT>::value || RequiresEvaluation<VT>::value };
122  //**********************************************************************************************
123 
124  //**********************************************************************************************
126  enum { evaluateMatrix = ( IsComputation<MT>::value && IsSame<MET,VET>::value &&
128  //**********************************************************************************************
129 
130  //**********************************************************************************************
132 
136  template< typename T1 >
137  struct UseSMPAssign {
138  enum { value = ( evaluateVector || evaluateMatrix ) };
139  };
141  //**********************************************************************************************
142 
143  //**********************************************************************************************
145 
149  template< typename T1, typename T2, typename T3 >
150  struct UseVectorizedKernel {
151  enum { value = useOptimizedKernels &&
152  !IsDiagonal<T3>::value &&
153  T1::vectorizable && T3::vectorizable &&
154  IsSame<typename T1::ElementType,typename T2::ElementType>::value &&
155  IsSame<typename T1::ElementType,typename T3::ElementType>::value &&
156  IntrinsicTrait<typename T1::ElementType>::addition &&
157  IntrinsicTrait<typename T1::ElementType>::multiplication };
158  };
160  //**********************************************************************************************
161 
162  //**********************************************************************************************
164 
168  template< typename T1, typename T2, typename T3 >
169  struct UseOptimizedKernel {
170  enum { value = useOptimizedKernels &&
171  !UseVectorizedKernel<T1,T2,T3>::value &&
172  !IsDiagonal<T3>::value &&
173  !IsResizable<typename T1::ElementType>::value &&
174  !IsResizable<VET>::value };
175  };
177  //**********************************************************************************************
178 
179  //**********************************************************************************************
181 
184  template< typename T1, typename T2, typename T3 >
185  struct UseDefaultKernel {
186  enum { value = !UseVectorizedKernel<T1,T2,T3>::value &&
187  !UseOptimizedKernel<T1,T2,T3>::value };
188  };
190  //**********************************************************************************************
191 
192  public:
193  //**Type definitions****************************************************************************
199  typedef const ElementType ReturnType;
200  typedef const ResultType CompositeType;
201 
203  typedef typename SelectType< IsExpression<VT>::value, const VT, const VT& >::Type LeftOperand;
204 
206  typedef typename SelectType< IsExpression<MT>::value, const MT, const MT& >::Type RightOperand;
207 
210 
213  //**********************************************************************************************
214 
215  //**Compilation flags***************************************************************************
217  enum { vectorizable = !IsDiagonal<MT>::value &&
218  MT::vectorizable &&
222 
224  enum { smpAssignable = !evaluateVector && VT::smpAssignable &&
225  !evaluateMatrix && MT::smpAssignable };
226  //**********************************************************************************************
227 
228  //**Constructor*********************************************************************************
234  explicit inline TSVecDMatMultExpr( const VT& vec, const MT& mat )
235  : vec_( vec ) // Left-hand side sparse vector of the multiplication expression
236  , mat_( mat ) // Right-hand side dense matrix of the multiplication expression
237  {
238  BLAZE_INTERNAL_ASSERT( vec_.size() == mat_.rows(), "Invalid vector and matrix sizes" );
239  }
240  //**********************************************************************************************
241 
242  //**Subscript operator**************************************************************************
248  inline ReturnType operator[]( size_t index ) const {
249  BLAZE_INTERNAL_ASSERT( index < mat_.columns(), "Invalid vector access index" );
250 
252 
253  LT x( vec_ ); // Evaluation of the left-hand side sparse vector operand
254 
255  BLAZE_INTERNAL_ASSERT( x.size() == vec_.size(), "Invalid vector size" );
256 
257  const ConstIterator end( ( IsUpper<MT>::value )
258  ?( IsStrictlyUpper<MT>::value ? x.lowerBound( index ) : x.upperBound( index ) )
259  :( x.end() ) );
260  ConstIterator element( ( IsLower<MT>::value )
261  ?( IsStrictlyLower<MT>::value ? x.upperBound( index ) : x.lowerBound( index ) )
262  :( x.begin() ) );
263 
264  ElementType res = ElementType();
265 
266  if( element != end ) {
267  res = element->value() * mat_( element->index(), index );
268  ++element;
269  for( ; element!=end; ++element ) {
270  res += element->value() * mat_( element->index(), index );
271  }
272  }
273 
274  return res;
275  }
276  //**********************************************************************************************
277 
278  //**At function*********************************************************************************
285  inline ReturnType at( size_t index ) const {
286  if( index >= mat_.columns() ) {
287  BLAZE_THROW_OUT_OF_RANGE( "Invalid vector access index" );
288  }
289  return (*this)[index];
290  }
291  //**********************************************************************************************
292 
293  //**Size function*******************************************************************************
298  inline size_t size() const {
299  return mat_.columns();
300  }
301  //**********************************************************************************************
302 
303  //**Left operand access*************************************************************************
308  inline LeftOperand leftOperand() const {
309  return vec_;
310  }
311  //**********************************************************************************************
312 
313  //**Right operand access************************************************************************
318  inline RightOperand rightOperand() const {
319  return mat_;
320  }
321  //**********************************************************************************************
322 
323  //**********************************************************************************************
329  template< typename T >
330  inline bool canAlias( const T* alias ) const {
331  return vec_.isAliased( alias ) || mat_.isAliased( alias );
332  }
333  //**********************************************************************************************
334 
335  //**********************************************************************************************
341  template< typename T >
342  inline bool isAliased( const T* alias ) const {
343  return vec_.isAliased( alias ) || mat_.isAliased( alias );
344  }
345  //**********************************************************************************************
346 
347  //**********************************************************************************************
352  inline bool isAligned() const {
353  return mat_.isAligned();
354  }
355  //**********************************************************************************************
356 
357  //**********************************************************************************************
362  inline bool canSMPAssign() const {
363  return ( size() > SMP_TSVECDMATMULT_THRESHOLD );
364  }
365  //**********************************************************************************************
366 
367  private:
368  //**Member variables****************************************************************************
369  LeftOperand vec_;
370  RightOperand mat_;
371  //**********************************************************************************************
372 
373  //**Assignment to dense vectors*****************************************************************
386  template< typename VT2 > // Type of the target dense vector
387  friend inline void assign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
388  {
390 
391  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
392 
393  // Evaluation of the left-hand side sparse vector operand
394  LT x( serial( rhs.vec_ ) );
395  if( x.nonZeros() == 0UL ) {
396  reset( ~lhs );
397  return;
398  }
399 
400  // Evaluation of the right-hand side dense matrix operand
401  RT A( serial( rhs.mat_ ) );
402 
403  // Checking the evaluated operands
404  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
405  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
406  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
407  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
408 
409  // Performing the sparse vector-dense matrix multiplication
410  TSVecDMatMultExpr::selectAssignKernel( ~lhs, x, A );
411  }
413  //**********************************************************************************************
414 
415  //**Default assignment to dense vectors*********************************************************
429  template< typename VT1 // Type of the left-hand side target vector
430  , typename VT2 // Type of the left-hand side vector operand
431  , typename MT1 > // Type of the right-hand side matrix operand
432  static inline typename EnableIf< UseDefaultKernel<VT1,VT2,MT1> >::Type
433  selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
434  {
436 
437  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
438 
439  const size_t N( A.columns() );
440 
441  ConstIterator element( x.begin() );
442  const ConstIterator end( x.end() );
443 
444  size_t last( 0UL );
445 
446  if( IsUpper<MT1>::value ) {
447  const size_t jend( IsStrictlyUpper<MT1>::value ? element->index()+1UL : element->index() );
448  for( size_t j=0UL; j<jend; ++j )
449  reset( y[j] );
450  }
451 
452  for( ; element!=end; ++element )
453  {
454  const size_t index( element->index() );
455 
456  if( IsDiagonal<MT1>::value )
457  {
458 
459 
460  for( size_t j=last; j<index; ++j )
461  reset( y[j] );
462 
463  y[index] = element->value() * A(index,index);
464  last = index + 1UL;
465  }
466  else
467  {
468  const size_t jbegin( ( IsUpper<MT1>::value )
469  ?( IsStrictlyUpper<MT1>::value ? index+1UL : index )
470  :( 0UL ) );
471  const size_t jend( ( IsLower<MT1>::value )
472  ?( IsStrictlyLower<MT1>::value ? index : index+1UL )
473  :( N ) );
474  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
475 
476  for( size_t j=jbegin; j<last; ++j ) {
477  y[j] += element->value() * A(index,j);
478  }
479  for( size_t j=last; j<jend; ++j ) {
480  y[j] = element->value() * A(index,j);
481  }
482 
483  last = jend;
484  }
485  }
486 
487  if( IsLower<MT1>::value ) {
488  for( size_t j=last; j<N; ++j )
489  reset( y[j] );
490  }
491  }
493  //**********************************************************************************************
494 
495  //**Optimized assignment to dense vectors*******************************************************
509  template< typename VT1 // Type of the left-hand side target vector
510  , typename VT2 // Type of the left-hand side vector operand
511  , typename MT1 > // Type of the right-hand side matrix operand
512  static inline typename EnableIf< UseOptimizedKernel<VT1,VT2,MT1> >::Type
513  selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
514  {
516 
517  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
518 
519  const size_t N( A.columns() );
520 
521  ConstIterator element( x.begin() );
522  const ConstIterator end( x.end() );
523 
524  const size_t ipos( x.nonZeros() & size_t(-4) );
525  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
526 
527  if( ipos > 3UL )
528  {
529  const size_t i1( element->index() );
530  const VET v1( element->value() );
531  ++element;
532  const size_t i2( element->index() );
533  const VET v2( element->value() );
534  ++element;
535  const size_t i3( element->index() );
536  const VET v3( element->value() );
537  ++element;
538  const size_t i4( element->index() );
539  const VET v4( element->value() );
540  ++element;
541 
542  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
543 
544  for( size_t j=0UL; j<N; ++j ) {
545  y[j] = v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
546  }
547  }
548  else
549  {
550  const size_t i1( element->index() );
551  const VET v1( element->value() );
552  ++element;
553 
554  for( size_t j=0UL; j<N; ++j ) {
555  y[j] = v1 * A(i1,j);
556  }
557  }
558 
559  for( size_t i=(ipos>3UL)?(4UL):(1UL); (i+4UL)<=ipos; i+=4UL )
560  {
561  const size_t i1( element->index() );
562  const VET v1( element->value() );
563  ++element;
564  const size_t i2( element->index() );
565  const VET v2( element->value() );
566  ++element;
567  const size_t i3( element->index() );
568  const VET v3( element->value() );
569  ++element;
570  const size_t i4( element->index() );
571  const VET v4( element->value() );
572  ++element;
573 
574  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
575 
576  const size_t jbegin( ( IsUpper<MT1>::value )
577  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
578  :( 0UL ) );
579  const size_t jend( ( IsLower<MT1>::value )
580  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
581  :( N ) );
582  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
583 
584  for( size_t j=jbegin; j<jend; ++j ) {
585  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
586  }
587  }
588  for( ; element!=end; ++element )
589  {
590  const size_t i1( element->index() );
591  const VET v1( element->value() );
592 
593  const size_t jbegin( ( IsUpper<MT1>::value )
594  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
595  :( 0UL ) );
596  const size_t jend( ( IsLower<MT1>::value )
597  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
598  :( N ) );
599  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
600 
601  for( size_t j=jbegin; j<jend; ++j ) {
602  y[j] += v1 * A(i1,j);
603  }
604  }
605  }
607  //**********************************************************************************************
608 
609  //**Vectorized assignment to dense vectors******************************************************
623  template< typename VT1 // Type of the left-hand side target vector
624  , typename VT2 // Type of the left-hand side vector operand
625  , typename MT1 > // Type of the right-hand side matrix operand
626  static inline typename EnableIf< UseVectorizedKernel<VT1,VT2,MT1> >::Type
627  selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
628  {
629  typedef IntrinsicTrait<ElementType> IT;
631 
632  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
633 
634  const size_t N( A.columns() );
635 
636  const bool remainder( !IsPadded<VT1>::value || !IsPadded<MT1>::value );
637 
638  ConstIterator element( x.begin() );
639  const ConstIterator end( x.end() );
640 
641  const size_t ipos( x.nonZeros() & size_t(-4) );
642  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
643 
644  if( ipos > 3UL )
645  {
646  const size_t i1( element->index() );
647  const VET v1( element->value() );
648  ++element;
649  const size_t i2( element->index() );
650  const VET v2( element->value() );
651  ++element;
652  const size_t i3( element->index() );
653  const VET v3( element->value() );
654  ++element;
655  const size_t i4( element->index() );
656  const VET v4( element->value() );
657  ++element;
658 
659  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
660 
661  const IntrinsicType xmm1( set( v1 ) );
662  const IntrinsicType xmm2( set( v2 ) );
663  const IntrinsicType xmm3( set( v3 ) );
664  const IntrinsicType xmm4( set( v4 ) );
665 
666  const size_t jpos( remainder ? ( N & size_t(-IT::size) ) : N );
667  BLAZE_INTERNAL_ASSERT( !remainder || ( N - ( N % IT::size ) ) == jpos, "Invalid end calculation" );
668 
669  size_t j( 0UL );
670 
671  for( ; j<jpos; j+=IT::size ) {
672  y.store( j, xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
673  }
674  for( ; remainder && j<N; ++j ) {
675  y[j] = v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
676  }
677  }
678  else
679  {
680  const size_t i1( element->index() );
681  const VET v1( element->value() );
682  ++element;
683 
684  const IntrinsicType xmm1( set( v1 ) );
685 
686  const size_t jpos( remainder ? ( N & size_t(-IT::size) ) : N );
687  BLAZE_INTERNAL_ASSERT( !remainder || ( N - ( N % IT::size ) ) == jpos, "Invalid end calculation" );
688 
689  size_t j( 0UL );
690 
691  for( ; j<jpos; j+=IT::size ) {
692  y.store( j, xmm1 * A.load(i1,j) );
693  }
694  for( ; remainder && j<N; ++j ) {
695  y[j] = v1 * A(i1,j);
696  }
697  }
698 
699  for( size_t i=(ipos>3UL)?(4UL):(1UL); (i+4UL)<=ipos; i+=4UL )
700  {
701  const size_t i1( element->index() );
702  const VET v1( element->value() );
703  ++element;
704  const size_t i2( element->index() );
705  const VET v2( element->value() );
706  ++element;
707  const size_t i3( element->index() );
708  const VET v3( element->value() );
709  ++element;
710  const size_t i4( element->index() );
711  const VET v4( element->value() );
712  ++element;
713 
714  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
715 
716  const IntrinsicType xmm1( set( v1 ) );
717  const IntrinsicType xmm2( set( v2 ) );
718  const IntrinsicType xmm3( set( v3 ) );
719  const IntrinsicType xmm4( set( v4 ) );
720 
721  const size_t jbegin( ( IsUpper<MT1>::value )
722  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
723  :( 0UL ) );
724  const size_t jend( ( IsLower<MT1>::value )
725  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
726  :( N ) );
727  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
728 
729  const size_t jpos( remainder ? ( jend & size_t(-IT::size) ) : jend );
730  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % IT::size ) ) == jpos, "Invalid end calculation" );
731 
732  size_t j( jbegin );
733 
734  for( ; j<jpos; j+=IT::size ) {
735  y.store( j, y.load(j) + xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
736  }
737  for( ; remainder && j<jend; ++j ) {
738  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
739  }
740  }
741  for( ; element!=end; ++element )
742  {
743  const size_t i1( element->index() );
744  const VET v1( element->value() );
745 
746  const IntrinsicType xmm1( set( v1 ) );
747 
748  const size_t jbegin( ( IsUpper<MT1>::value )
749  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
750  :( 0UL ) );
751  const size_t jend( ( IsLower<MT1>::value )
752  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
753  :( N ) );
754  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
755 
756  const size_t jpos( remainder ? ( jend & size_t(-IT::size) ) : jend );
757  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % IT::size ) ) == jpos, "Invalid end calculation" );
758 
759  size_t j( jbegin );
760 
761  for( ; j<jpos; j+=IT::size ) {
762  y.store( j, y.load(j) + xmm1 * A.load(i1,j) );
763  }
764  for( ; remainder && j<jend; ++j ) {
765  y[j] += v1 * A(i1,j);
766  }
767  }
768  }
770  //**********************************************************************************************
771 
772  //**Assignment to sparse vectors****************************************************************
785  template< typename VT2 > // Type of the target sparse vector
786  friend inline void assign( SparseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
787  {
789 
793 
794  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
795 
796  const ResultType tmp( serial( rhs ) );
797  assign( ~lhs, tmp );
798  }
800  //**********************************************************************************************
801 
802  //**Addition assignment to dense vectors********************************************************
814  template< typename VT2 > // Type of the target dense vector
815  friend inline void addAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
816  {
818 
819  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
820 
821  // Evaluation of the left-hand side sparse vector operand
822  LT x( serial( rhs.vec_ ) );
823  if( x.nonZeros() == 0UL ) return;
824 
825  // Evaluation of the right-hand side dense matrix operand
826  RT A( serial( rhs.mat_ ) );
827 
828  // Checking the evaluated operands
829  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
830  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
831  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
832  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
833 
834  // Performing the sparse vector-dense matrix multiplication
835  TSVecDMatMultExpr::selectAddAssignKernel( ~lhs, x, A );
836  }
837  //**********************************************************************************************
838 
839  //**Default addition assignment to dense vectors************************************************
853  template< typename VT1 // Type of the left-hand side target vector
854  , typename VT2 // Type of the left-hand side vector operand
855  , typename MT1 > // Type of the right-hand side matrix operand
856  static inline typename EnableIf< UseDefaultKernel<VT1,VT2,MT1> >::Type
857  selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
858  {
860 
861  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
862 
863  const size_t N( A.columns() );
864 
865  ConstIterator element( x.begin() );
866  const ConstIterator end( x.end() );
867 
868  for( ; element!=end; ++element )
869  {
870  const size_t index( element->index() );
871 
873  {
874  y[index] += A(index,index) * element->value();
875  }
876  else
877  {
878  const size_t jbegin( ( IsUpper<MT1>::value )
879  ?( IsStrictlyUpper<MT1>::value ? index+1UL : index )
880  :( 0UL ) );
881  const size_t jend( ( IsLower<MT1>::value )
882  ?( IsStrictlyLower<MT1>::value ? index : index+1UL )
883  :( N ) );
884  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
885 
886  for( size_t j=jbegin; j<jend; ++j ) {
887  y[j] += element->value() * A(index,j);
888  }
889  }
890  }
891  }
893  //**********************************************************************************************
894 
895  //**Optimized addition assignment to dense vectors**********************************************
909  template< typename VT1 // Type of the left-hand side target vector
910  , typename VT2 // Type of the left-hand side vector operand
911  , typename MT1 > // Type of the right-hand side matrix operand
912  static inline typename EnableIf< UseOptimizedKernel<VT1,VT2,MT1> >::Type
913  selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
914  {
916 
917  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
918 
919  const size_t N( A.columns() );
920 
921  ConstIterator element( x.begin() );
922  const ConstIterator end( x.end() );
923 
924  const size_t ipos( x.nonZeros() & size_t(-4) );
925  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
926 
927  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
928  {
929  const size_t i1( element->index() );
930  const VET v1( element->value() );
931  ++element;
932  const size_t i2( element->index() );
933  const VET v2( element->value() );
934  ++element;
935  const size_t i3( element->index() );
936  const VET v3( element->value() );
937  ++element;
938  const size_t i4( element->index() );
939  const VET v4( element->value() );
940  ++element;
941 
942  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
943 
944  const size_t jbegin( ( IsUpper<MT1>::value )
945  ?( IsStrictlyUpper<MT1>::value ? i+1UL : i1 )
946  :( 0UL ) );
947  const size_t jend( ( IsLower<MT1>::value )
948  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
949  :( N ) );
950  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
951 
952  for( size_t j=jbegin; j<jend; ++j ) {
953  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
954  }
955  }
956  for( ; element!=end; ++element )
957  {
958  const size_t i1( element->index() );
959  const VET v1( element->value() );
960 
961  const size_t jbegin( ( IsUpper<MT1>::value )
962  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
963  :( 0UL ) );
964  const size_t jend( ( IsLower<MT1>::value )
965  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
966  :( N ) );
967  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
968 
969  for( size_t j=jbegin; j<jend; ++j ) {
970  y[j] += v1 * A(i1,j);
971  }
972  }
973  }
975  //**********************************************************************************************
976 
977  //**Vectorized addition assignment to dense vectors*********************************************
991  template< typename VT1 // Type of the left-hand side target vector
992  , typename VT2 // Type of the left-hand side vector operand
993  , typename MT1 > // Type of the right-hand side matrix operand
994  static inline typename EnableIf< UseVectorizedKernel<VT1,VT2,MT1> >::Type
995  selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
996  {
997  typedef IntrinsicTrait<ElementType> IT;
999 
1000  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1001 
1002  const size_t N( A.columns() );
1003 
1004  const bool remainder( !IsPadded<VT1>::value || !IsPadded<MT1>::value );
1005 
1006  ConstIterator element( x.begin() );
1007  const ConstIterator end( x.end() );
1008 
1009  const size_t ipos( x.nonZeros() & size_t(-4) );
1010  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1011 
1012  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1013  {
1014  const size_t i1( element->index() );
1015  const VET v1( element->value() );
1016  ++element;
1017  const size_t i2( element->index() );
1018  const VET v2( element->value() );
1019  ++element;
1020  const size_t i3( element->index() );
1021  const VET v3( element->value() );
1022  ++element;
1023  const size_t i4( element->index() );
1024  const VET v4( element->value() );
1025  ++element;
1026 
1027  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1028 
1029  const IntrinsicType xmm1( set( v1 ) );
1030  const IntrinsicType xmm2( set( v2 ) );
1031  const IntrinsicType xmm3( set( v3 ) );
1032  const IntrinsicType xmm4( set( v4 ) );
1033 
1034  const size_t jbegin( ( IsUpper<MT1>::value )
1035  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
1036  :( 0UL ) );
1037  const size_t jend( ( IsLower<MT1>::value )
1038  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
1039  :( N ) );
1040  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1041 
1042  const size_t jpos( remainder ? ( jend & size_t(-IT::size) ) : jend );
1043  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % IT::size ) ) == jpos, "Invalid end calculation" );
1044 
1045  size_t j( jbegin );
1046 
1047  for( ; j<jpos; j+=IT::size ) {
1048  y.store( j, y.load(j) + xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
1049  }
1050  for( ; remainder && j<jend; ++j ) {
1051  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1052  }
1053  }
1054  for( ; element!=end; ++element )
1055  {
1056  const size_t i1( element->index() );
1057  const VET v1( element->value() );
1058 
1059  const IntrinsicType xmm1( set( v1 ) );
1060 
1061  const size_t jbegin( ( IsUpper<MT1>::value )
1062  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
1063  :( 0UL ) );
1064  const size_t jend( ( IsLower<MT1>::value )
1065  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
1066  :( N ) );
1067  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1068 
1069  const size_t jpos( remainder ? ( jend & size_t(-IT::size) ) : jend );
1070  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % IT::size ) ) == jpos, "Invalid end calculation" );
1071 
1072  size_t j( jbegin );
1073 
1074  for( ; j<jpos; j+=IT::size ) {
1075  y.store( j, y.load(j) + xmm1 * A.load(i1,j) );
1076  }
1077  for( ; remainder && j<jend; ++j ) {
1078  y[j] += v1 * A(i1,j);
1079  }
1080  }
1081  }
1083  //**********************************************************************************************
1084 
1085  //**Addition assignment to sparse vectors*******************************************************
1086  // No special implementation for the addition assignment to sparse vectors.
1087  //**********************************************************************************************
1088 
1089  //**Subtraction assignment to dense vectors*****************************************************
1101  template< typename VT2 > // Type of the target dense vector
1102  friend inline void subAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1103  {
1105 
1106  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1107 
1109 
1110  // Evaluation of the left-hand side sparse vector operand
1111  LT x( serial( rhs.vec_ ) );
1112  if( x.nonZeros() == 0UL ) return;
1113 
1114  // Evaluation of the right-hand side dense matrix operand
1115  RT A( serial( rhs.mat_ ) );
1116 
1117  // Checking the evaluated operands
1118  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1119  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1120  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1121  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1122 
1123  // Performing the sparse vector-dense matrix multiplication
1124  TSVecDMatMultExpr::selectSubAssignKernel( ~lhs, x, A );
1125  }
1126  //**********************************************************************************************
1127 
1128  //**Default subtraction assignment to dense vectors*********************************************
1142  template< typename VT1 // Type of the left-hand side target vector
1143  , typename VT2 // Type of the left-hand side vector operand
1144  , typename MT1 > // Type of the right-hand side matrix operand
1145  static inline typename EnableIf< UseDefaultKernel<VT1,VT2,MT1> >::Type
1146  selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1147  {
1149 
1150  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1151 
1152  const size_t N( A.columns() );
1153 
1154  ConstIterator element( x.begin() );
1155  const ConstIterator end( x.end() );
1156 
1157  for( ; element!=end; ++element )
1158  {
1159  const size_t index( element->index() );
1160 
1162  {
1163  y[index] -= A(index,index) * element->value();
1164  }
1165  else
1166  {
1167  const size_t jbegin( ( IsUpper<MT1>::value )
1168  ?( IsStrictlyUpper<MT1>::value ? index+1UL : index )
1169  :( 0UL ) );
1170  const size_t jend( ( IsLower<MT1>::value )
1171  ?( IsStrictlyLower<MT1>::value ? index : index+1UL )
1172  :( N ) );
1173  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1174 
1175  for( size_t j=jbegin; j<jend; ++j ) {
1176  y[j] -= element->value() * A(index,j);
1177  }
1178  }
1179  }
1180  }
1182  //**********************************************************************************************
1183 
1184  //**Optimized subtraction assignment to dense vectors*******************************************
1198  template< typename VT1 // Type of the left-hand side target vector
1199  , typename VT2 // Type of the left-hand side vector operand
1200  , typename MT1 > // Type of the right-hand side matrix operand
1201  static inline typename EnableIf< UseOptimizedKernel<VT1,VT2,MT1> >::Type
1202  selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1203  {
1205 
1206  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1207 
1208  const size_t N( A.columns() );
1209 
1210  ConstIterator element( x.begin() );
1211  const ConstIterator end( x.end() );
1212 
1213  const size_t ipos( x.nonZeros() & size_t(-4) );
1214  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1215 
1216  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1217  {
1218  const size_t i1( element->index() );
1219  const VET v1( element->value() );
1220  ++element;
1221  const size_t i2( element->index() );
1222  const VET v2( element->value() );
1223  ++element;
1224  const size_t i3( element->index() );
1225  const VET v3( element->value() );
1226  ++element;
1227  const size_t i4( element->index() );
1228  const VET v4( element->value() );
1229  ++element;
1230 
1231  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1232 
1233  const size_t jbegin( ( IsUpper<MT1>::value )
1234  ?( IsStrictlyUpper<MT1>::value ? i+1UL : i1 )
1235  :( 0UL ) );
1236  const size_t jend( ( IsLower<MT1>::value )
1237  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
1238  :( N ) );
1239  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1240 
1241  for( size_t j=jbegin; j<jend; ++j ) {
1242  y[j] -= v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1243  }
1244  }
1245  for( ; element!=end; ++element )
1246  {
1247  const size_t i1( element->index() );
1248  const VET v1( element->value() );
1249 
1250  const size_t jbegin( ( IsUpper<MT1>::value )
1251  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
1252  :( 0UL ) );
1253  const size_t jend( ( IsLower<MT1>::value )
1254  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
1255  :( N ) );
1256  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1257 
1258  for( size_t j=jbegin; j<jend; ++j ) {
1259  y[j] -= v1 * A(i1,j);
1260  }
1261  }
1262  }
1264  //**********************************************************************************************
1265 
1266  //**Vectorized subtraction assignment to dense vectors******************************************
1280  template< typename VT1 // Type of the left-hand side target vector
1281  , typename VT2 // Type of the left-hand side vector operand
1282  , typename MT1 > // Type of the right-hand side matrix operand
1283  static inline typename EnableIf< UseVectorizedKernel<VT1,VT2,MT1> >::Type
1284  selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1285  {
1286  typedef IntrinsicTrait<ElementType> IT;
1288 
1289  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1290 
1291  const size_t N( A.columns() );
1292 
1293  const bool remainder( !IsPadded<VT1>::value || !IsPadded<MT1>::value );
1294 
1295  ConstIterator element( x.begin() );
1296  const ConstIterator end( x.end() );
1297 
1298  const size_t ipos( x.nonZeros() & size_t(-4) );
1299  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1300 
1301  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1302  {
1303  const size_t i1( element->index() );
1304  const VET v1( element->value() );
1305  ++element;
1306  const size_t i2( element->index() );
1307  const VET v2( element->value() );
1308  ++element;
1309  const size_t i3( element->index() );
1310  const VET v3( element->value() );
1311  ++element;
1312  const size_t i4( element->index() );
1313  const VET v4( element->value() );
1314  ++element;
1315 
1316  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1317 
1318  const IntrinsicType xmm1( set( v1 ) );
1319  const IntrinsicType xmm2( set( v2 ) );
1320  const IntrinsicType xmm3( set( v3 ) );
1321  const IntrinsicType xmm4( set( v4 ) );
1322 
1323  const size_t jbegin( ( IsUpper<MT1>::value )
1324  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
1325  :( 0UL ) );
1326  const size_t jend( ( IsLower<MT1>::value )
1327  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
1328  :( N ) );
1329  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1330 
1331  const size_t jpos( remainder ? ( jend & size_t(-IT::size) ) : jend );
1332  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % IT::size ) ) == jpos, "Invalid end calculation" );
1333 
1334  size_t j( jbegin );
1335 
1336  for( ; j<jpos; j+=IT::size ) {
1337  y.store( j, y.load(j) - xmm1 * A.load(i1,j) - xmm2 * A.load(i2,j) - xmm3 * A.load(i3,j) - xmm4 * A.load(i4,j) );
1338  }
1339  for( ; remainder && j<jend; ++j ) {
1340  y[j] -= v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1341  }
1342  }
1343  for( ; element!=x.end(); ++element )
1344  {
1345  const size_t i1( element->index() );
1346  const VET v1( element->value() );
1347 
1348  const IntrinsicType xmm1( set( v1 ) );
1349 
1350  const size_t jbegin( ( IsUpper<MT1>::value )
1351  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-IT::size) )
1352  :( 0UL ) );
1353  const size_t jend( ( IsLower<MT1>::value )
1354  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
1355  :( N ) );
1356  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1357 
1358  const size_t jpos( remainder ? ( jend & size_t(-IT::size) ) : jend );
1359  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % IT::size ) ) == jpos, "Invalid end calculation" );
1360 
1361  size_t j( jbegin );
1362 
1363  for( ; j<jpos; j+=IT::size ) {
1364  y.store( j, y.load(j) - xmm1 * A.load(i1,j) );
1365  }
1366  for( ; remainder && j<jend; ++j ) {
1367  y[j] -= v1 * A(i1,j);
1368  }
1369  }
1370  }
1372  //**********************************************************************************************
1373 
1374  //**Subtraction assignment to sparse vectors****************************************************
1375  // No special implementation for the subtraction assignment to sparse vectors.
1376  //**********************************************************************************************
1377 
1378  //**Multiplication assignment to dense vectors**************************************************
1390  template< typename VT2 > // Type of the target dense vector
1391  friend inline void multAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1392  {
1394 
1398 
1399  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1400 
1401  const ResultType tmp( serial( rhs ) );
1402  multAssign( ~lhs, tmp );
1403  }
1404  //**********************************************************************************************
1405 
1406  //**Multiplication assignment to sparse vectors*************************************************
1407  // No special implementation for the multiplication assignment to sparse vectors.
1408  //**********************************************************************************************
1409 
1410  //**SMP assignment to dense vectors*************************************************************
1425  template< typename VT2 > // Type of the target dense vector
1426  friend inline typename EnableIf< UseSMPAssign<VT2> >::Type
1428  {
1430 
1431  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1432 
1433  // Evaluation of the left-hand side sparse vector operand
1434  LT x( rhs.vec_ );
1435  if( x.nonZeros() == 0UL ) {
1436  reset( ~lhs );
1437  return;
1438  }
1439 
1440  // Evaluation of the right-hand side dense matrix operand
1441  RT A( rhs.mat_ );
1442 
1443  // Checking the evaluated operands
1444  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1445  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1446  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1447  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1448 
1449  // Performing the sparse vector-dense matrix multiplication
1450  smpAssign( ~lhs, x * A );
1451  }
1453  //**********************************************************************************************
1454 
1455  //**SMP assignment to sparse vectors************************************************************
1470  template< typename VT2 > // Type of the target sparse vector
1471  friend inline typename EnableIf< UseSMPAssign<VT2> >::Type
1472  smpAssign( SparseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1473  {
1475 
1479 
1480  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1481 
1482  const ResultType tmp( rhs );
1483  smpAssign( ~lhs, tmp );
1484  }
1486  //**********************************************************************************************
1487 
1488  //**SMP addition assignment to dense vectors****************************************************
1502  template< typename VT2 > // Type of the target dense vector
1503  friend inline typename EnableIf< UseSMPAssign<VT2> >::Type
1505  {
1507 
1508  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1509 
1510  // Evaluation of the left-hand side sparse vector operand
1511  LT x( rhs.vec_ );
1512  if( x.nonZeros() == 0UL ) return;
1513 
1514  // Evaluation of the right-hand side dense matrix operand
1515  RT A( rhs.mat_ );
1516 
1517  // Checking the evaluated operands
1518  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1519  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1520  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1521  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1522 
1523  // Performing the sparse vector-dense matrix multiplication
1524  smpAddAssign( ~lhs, x * A );
1525  }
1526  //**********************************************************************************************
1527 
1528  //**SMP addition assignment to sparse vectors***************************************************
1529  // No special implementation for the SMP addition assignment to sparse vectors.
1530  //**********************************************************************************************
1531 
1532  //**SMP subtraction assignment to dense vectors*************************************************
1546  template< typename VT2 > // Type of the target dense vector
1547  friend inline typename EnableIf< UseSMPAssign<VT2> >::Type
1549  {
1551 
1552  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1553 
1555 
1556  // Evaluation of the left-hand side sparse vector operand
1557  LT x( rhs.vec_ );
1558  if( x.nonZeros() == 0UL ) return;
1559 
1560  // Evaluation of the right-hand side dense matrix operand
1561  RT A( rhs.mat_ );
1562 
1563  // Checking the evaluated operands
1564  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1565  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1566  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1567  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1568 
1569  // Performing the sparse vector-dense matrix multiplication
1570  smpSubAssign( ~lhs, x * A );
1571  }
1572  //**********************************************************************************************
1573 
1574  //**SMP subtraction assignment to sparse vectors************************************************
1575  // No special implementation for the SMP subtraction assignment to sparse vectors.
1576  //**********************************************************************************************
1577 
1578  //**SMP multiplication assignment to dense vectors**********************************************
1592  template< typename VT2 > // Type of the target dense vector
1593  friend inline typename EnableIf< UseSMPAssign<VT2> >::Type
1595  {
1597 
1601 
1602  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1603 
1604  const ResultType tmp( rhs );
1605  smpMultAssign( ~lhs, tmp );
1606  }
1607  //**********************************************************************************************
1608 
1609  //**SMP multiplication assignment to sparse vectors*********************************************
1610  // No special implementation for the SMP multiplication assignment to sparse vectors.
1611  //**********************************************************************************************
1612 
1613  //**Compile time checks*************************************************************************
1621  //**********************************************************************************************
1622 };
1623 //*************************************************************************************************
1624 
1625 
1626 
1627 
1628 //=================================================================================================
1629 //
1630 // GLOBAL BINARY ARITHMETIC OPERATORS
1631 //
1632 //=================================================================================================
1633 
1634 //*************************************************************************************************
1665 template< typename T1, typename T2 >
1666 inline const typename DisableIf< IsMatMatMultExpr<T2>, TSVecDMatMultExpr<T1,T2> >::Type
1668 {
1670 
1671  if( (~vec).size() != (~mat).rows() ) {
1672  BLAZE_THROW_INVALID_ARGUMENT( "Vector and matrix sizes do not match" );
1673  }
1674 
1675  return TSVecDMatMultExpr<T1,T2>( ~vec, ~mat );
1676 }
1677 //*************************************************************************************************
1678 
1679 
1680 
1681 
1682 //=================================================================================================
1683 //
1684 // GLOBAL RESTRUCTURING BINARY ARITHMETIC OPERATORS
1685 //
1686 //=================================================================================================
1687 
1688 //*************************************************************************************************
1701 template< typename T1 // Type of the left-hand side sparse vector
1702  , typename T2 // Type of the right-hand side dense matrix
1703  , bool SO > // Storage order of the right-hand side dense matrix
1704 inline const typename EnableIf< IsMatMatMultExpr<T2>, typename MultExprTrait<T1,T2>::Type >::Type
1706 {
1708 
1710 
1711  return ( vec * (~mat).leftOperand() ) * (~mat).rightOperand();
1712 }
1713 //*************************************************************************************************
1714 
1715 
1716 
1717 
1718 //=================================================================================================
1719 //
1720 // SIZE SPECIALIZATIONS
1721 //
1722 //=================================================================================================
1723 
1724 //*************************************************************************************************
1726 template< typename VT, typename MT >
1727 struct Size< TSVecDMatMultExpr<VT,MT> > : public Columns<MT>
1728 {};
1730 //*************************************************************************************************
1731 
1732 
1733 
1734 
1735 //=================================================================================================
1736 //
1737 // ISALIGNED SPECIALIZATIONS
1738 //
1739 //=================================================================================================
1740 
1741 //*************************************************************************************************
1743 template< typename VT, typename MT >
1744 struct IsAligned< TSVecDMatMultExpr<VT,MT> > : public IsTrue< IsAligned<MT>::value >
1745 {};
1747 //*************************************************************************************************
1748 
1749 
1750 
1751 
1752 //=================================================================================================
1753 //
1754 // EXPRESSION TRAIT SPECIALIZATIONS
1755 //
1756 //=================================================================================================
1757 
1758 //*************************************************************************************************
1760 template< typename VT, typename MT, bool AF >
1761 struct SubvectorExprTrait< TSVecDMatMultExpr<VT,MT>, AF >
1762 {
1763  public:
1764  //**********************************************************************************************
1765  typedef typename MultExprTrait< typename SubvectorExprTrait<const VT,AF>::Type
1766  , typename SubmatrixExprTrait<const MT,AF>::Type >::Type Type;
1767  //**********************************************************************************************
1768 };
1770 //*************************************************************************************************
1771 
1772 } // namespace blaze
1773 
1774 #endif
IntrinsicTrait< ElementType >::Type IntrinsicType
Resulting intrinsic element type.
Definition: TSVecDMatMultExpr.h:198
bool isAliased(const T *alias) const
Returns whether the expression is aliased with the given address alias.
Definition: TSVecDMatMultExpr.h:342
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exceptionThis macro encapsulates the default way of...
Definition: Exception.h:187
BLAZE_ALWAYS_INLINE EnableIf< And< IsIntegral< T >, HasSize< T, 2UL > >, simd_int16_t >::Type set(T value)
Sets all values in the vector to the given 2-byte integral value.
Definition: Set.h:73
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:89
friend void multAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Multiplication assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ...
Definition: TSVecDMatMultExpr.h:1391
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:7820
Header file for basic type definitions.
TSVecDMatMultExpr< VT, MT > This
Type of this TSVecDMatMultExpr instance.
Definition: TSVecDMatMultExpr.h:194
Expression object for transpose sparse vector-dense matrix multiplications.The TSVecDMatMultExpr clas...
Definition: Forward.h:160
SelectType< IsExpression< VT >::value, const VT, const VT & >::Type LeftOperand
Composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:203
BLAZE_ALWAYS_INLINE size_t size(const Vector< VT, TF > &vector)
Returns the current size/dimension of the vector.
Definition: Vector.h:252
SelectType< IsExpression< MT >::value, const MT, const MT & >::Type RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TSVecDMatMultExpr.h:206
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:250
Efficient implementation of a compressed matrix.The CompressedMatrix class template is the represent...
Definition: CompressedMatrix.h:207
Header file for the IsDiagonal type trait.
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:79
Header file for the IsSame and IsStrictlySame type traits.
LeftOperand vec_
Left-hand side sparse vector of the multiplication expression.
Definition: TSVecDMatMultExpr.h:369
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:507
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:2588
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:259
BLAZE_ALWAYS_INLINE size_t rows(const Matrix< MT, SO > &matrix)
Returns the current number of rows of the matrix.
Definition: Matrix.h:308
Header file for the DenseVector base class.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:90
ReturnType at(size_t index) const
Checked access to the vector elements.
Definition: TSVecDMatMultExpr.h:285
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:721
Header file for the Computation base class.
Type relationship analysis.This class tests if the two data types A and B are equal. For this type comparison, the cv-qualifiers of both data types are ignored. If A and B are the same data type (ignoring the cv-qualifiers), then the value member enumeration is set to 1, the nested type definition Type is TrueType, and the class derives from TrueType. Otherwise value is set to 0, Type is FalseType, and the class derives from FalseType.
Definition: IsSame.h:158
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:90
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
MRT::ElementType MET
Element type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:114
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
VRT::ElementType VET
Element type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:113
Constraint on the data type.
bool isAligned() const
Returns whether the operands of the expression are properly aligned in memory.
Definition: TSVecDMatMultExpr.h:352
bool canSMPAssign() const
Returns whether the expression can be used in SMP assignments.
Definition: TSVecDMatMultExpr.h:362
ResultType::TransposeType TransposeType
Transpose type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:196
Constraint on the data type.
Header file for the MultExprTrait class template.
RightOperand mat_
Right-hand side dense matrix of the multiplication expression.
Definition: TSVecDMatMultExpr.h:370
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:90
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:261
VT::CompositeType VCT
Composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:115
Compile time type selection.The SelectType class template selects one of the two given types T1 and T...
Definition: SelectType.h:59
Header file for the DisableIf class template.
friend void addAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Addition assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:815
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
MT::CompositeType MCT
Composite type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:116
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2592
Header file for the IsMatMatMultExpr type trait class.
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exceptionThis macro encapsulates the default way of Bla...
Definition: Exception.h:331
Header file for the Columns type trait.
Base class for N-dimensional dense vectors.The DenseVector class is a base class for all arbitrarily ...
Definition: DenseVector.h:70
Header file for the IsLower type trait.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional vector type...
Definition: SparseVector.h:79
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:92
SelectType< evaluateVector, const VRT, VCT >::Type LT
Type for the assignment of the left-hand side sparse vector operand.
Definition: TSVecDMatMultExpr.h:209
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:78
bool canAlias(const T *alias) const
Returns whether the expression can alias with the given address alias.
Definition: TSVecDMatMultExpr.h:330
Constraint on the data type.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
Type ElementType
Type of the sparse matrix elements.
Definition: CompressedMatrix.h:2586
Header file for the SelectType class template.
Header file for all forward declarations for expression class templates.
Constraint on the data type.
TSVecDMatMultExpr(const VT &vec, const MT &mat)
Constructor for the TSVecDMatMultExpr class.
Definition: TSVecDMatMultExpr.h:234
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
Header file for the serial shim.
SelectType< evaluateMatrix, const MRT, MCT >::Type RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: TSVecDMatMultExpr.h:212
friend EnableIf< UseSMPAssign< VT2 > >::Type smpMultAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP multiplication assignment of a transpose sparse vector-dense matrix multiplication to a dense vec...
Definition: TSVecDMatMultExpr.h:1594
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type, a compilation error is created.
Definition: Symmetric.h:116
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:79
Removal of reference modifiers.The RemoveCV type trait removes any reference modifiers from the given...
Definition: RemoveReference.h:69
ResultType::ElementType ElementType
Resulting element type.
Definition: TSVecDMatMultExpr.h:197
Intrinsic characteristics of data types.The IntrinsicTrait class template provides the intrinsic char...
Definition: IntrinsicTrait.h:1232
Header file for run time assertion macros.
EnableIf< IsDenseMatrix< MT1 > >::Type smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:199
Base template for the MultTrait class.
Definition: MultTrait.h:138
friend EnableIf< UseSMPAssign< VT2 > >::Type smpAddAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP addition assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ...
Definition: TSVecDMatMultExpr.h:1504
LeftOperand leftOperand() const
Returns the left-hand side sparse vector operand.
Definition: TSVecDMatMultExpr.h:308
Substitution Failure Is Not An Error (SFINAE) class.The EnableIf class template is an auxiliary tool ...
Definition: EnableIf.h:184
const bool useOptimizedKernels
Configuration switch for optimized kernels.This configuration switch enables/disables all optimized c...
Definition: Optimizations.h:84
Header file for the reset shim.
Constraint on the data type.
size_t size() const
Returns the current size/dimension of the vector.
Definition: TSVecDMatMultExpr.h:298
Compile time check for data types.This type trait tests whether or not the given template parameter i...
Definition: IsBlasCompatible.h:99
ReturnType operator[](size_t index) const
Subscript operator for the direct access to the vector elements.
Definition: TSVecDMatMultExpr.h:248
Header file for the TVecMatMultExpr base class.
Constraint on the data type.
MultTrait< VRT, MRT >::Type ResultType
Result type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:195
Header file for the RemoveReference type trait.
Header file for all intrinsic functionality.
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional vector type...
Definition: DenseVector.h:79
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_TVECMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid vector/matrix ...
Definition: TVecMatMultExpr.h:166
Header file for the IsComputation type trait class.
friend EnableIf< UseSMPAssign< VT2 > >::Type smpSubAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP subtraction assignment of a transpose sparse vector-dense matrix multiplication to a dense vector...
Definition: TSVecDMatMultExpr.h:1548
RightOperand rightOperand() const
Returns the right-hand side dense matrix operand.
Definition: TSVecDMatMultExpr.h:318
Base class for sparse vectors.The SparseVector class is a base class for all arbitrarily sized (N-dim...
Definition: Forward.h:118
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
This ResultType
Result type for expression template evaluations.
Definition: CompressedMatrix.h:2583
#define BLAZE_CONSTRAINT_MUST_BE_ROW_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a row dense or sparse vector type (i...
Definition: RowVector.h:79
Header file for the SubvectorExprTrait class template.
friend void subAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Subtraction assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:1102
Header file for the IsUpper type trait.
Header file for exception macros.
Header file for the IsResizable type trait.
const ResultType CompositeType
Data type for composite expression templates.
Definition: TSVecDMatMultExpr.h:200
Header file for the Size type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
VT::ResultType VRT
Result type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:111
Constraint on the transpose flag of vector types.
MT::ResultType MRT
Result type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:112
Header file for the IsExpression type trait class.
Header file for the FunctionTrace class.