TSVecDMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TSVECDMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TSVECDMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
51 #include <blaze/math/Exception.h>
56 #include <blaze/math/shims/Reset.h>
58 #include <blaze/math/SIMD.h>
81 #include <blaze/util/Assert.h>
83 #include <blaze/util/DisableIf.h>
84 #include <blaze/util/EnableIf.h>
87 #include <blaze/util/mpl/If.h>
88 #include <blaze/util/Types.h>
91 
92 
93 namespace blaze {
94 
95 //=================================================================================================
96 //
97 // CLASS TSVECDMATMULTEXPR
98 //
99 //=================================================================================================
100 
101 //*************************************************************************************************
108 template< typename VT // Type of the left-hand side sparse vector
109  , typename MT > // Type of the right-hand side dense matrix
110 class TSVecDMatMultExpr : public DenseVector< TSVecDMatMultExpr<VT,MT>, true >
111  , private TVecMatMultExpr
112  , private Computation
113 {
114  private:
115  //**Type definitions****************************************************************************
122  //**********************************************************************************************
123 
124  //**********************************************************************************************
126  enum : bool { evaluateVector = IsComputation<VT>::value || RequiresEvaluation<VT>::value };
127  //**********************************************************************************************
128 
129  //**********************************************************************************************
131  enum : bool { evaluateMatrix = RequiresEvaluation<MT>::value };
132  //**********************************************************************************************
133 
134  //**********************************************************************************************
136 
140  template< typename T1 >
141  struct UseSMPAssign {
142  enum : bool { value = ( evaluateVector || evaluateMatrix ) };
143  };
145  //**********************************************************************************************
146 
147  //**********************************************************************************************
149 
153  template< typename T1, typename T2, typename T3 >
154  struct UseVectorizedKernel {
155  enum : bool { value = useOptimizedKernels &&
156  !IsDiagonal<T3>::value &&
157  T1::simdEnabled && T3::simdEnabled &&
158  AreSIMDCombinable< ElementType_<T1>
159  , ElementType_<T2>
160  , ElementType_<T3> >::value &&
161  HasSIMDAdd< ElementType_<T2>, ElementType_<T3> >::value &&
162  HasSIMDMult< ElementType_<T2>, ElementType_<T3> >::value };
163  };
165  //**********************************************************************************************
166 
167  //**********************************************************************************************
169 
173  template< typename T1, typename T2, typename T3 >
174  struct UseOptimizedKernel {
175  enum : bool { value = useOptimizedKernels &&
176  !UseVectorizedKernel<T1,T2,T3>::value &&
177  !IsDiagonal<T3>::value &&
178  !IsResizable< ElementType_<T1> >::value &&
179  !IsResizable<VET>::value };
180  };
182  //**********************************************************************************************
183 
184  //**********************************************************************************************
186 
189  template< typename T1, typename T2, typename T3 >
190  struct UseDefaultKernel {
191  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
192  !UseOptimizedKernel<T1,T2,T3>::value };
193  };
195  //**********************************************************************************************
196 
197  public:
198  //**Type definitions****************************************************************************
204  typedef const ElementType ReturnType;
205  typedef const ResultType CompositeType;
206 
208  typedef If_< IsExpression<VT>, const VT, const VT& > LeftOperand;
209 
211  typedef If_< IsExpression<MT>, const MT, const MT& > RightOperand;
212 
215 
218  //**********************************************************************************************
219 
220  //**Compilation flags***************************************************************************
222  enum : bool { simdEnabled = !IsDiagonal<MT>::value &&
223  MT::simdEnabled &&
226 
228  enum : bool { smpAssignable = !evaluateVector && VT::smpAssignable &&
229  !evaluateMatrix && MT::smpAssignable };
230  //**********************************************************************************************
231 
232  //**SIMD properties*****************************************************************************
234  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
235  //**********************************************************************************************
236 
237  //**Constructor*********************************************************************************
243  explicit inline TSVecDMatMultExpr( const VT& vec, const MT& mat ) noexcept
244  : vec_( vec ) // Left-hand side sparse vector of the multiplication expression
245  , mat_( mat ) // Right-hand side dense matrix of the multiplication expression
246  {
247  BLAZE_INTERNAL_ASSERT( vec_.size() == mat_.rows(), "Invalid vector and matrix sizes" );
248  }
249  //**********************************************************************************************
250 
251  //**Subscript operator**************************************************************************
257  inline ReturnType operator[]( size_t index ) const {
258  BLAZE_INTERNAL_ASSERT( index < mat_.columns(), "Invalid vector access index" );
259 
261  {
262  return vec_[index] * mat_(index,index);
263  }
264  else if( IsLower<MT>::value )
265  {
266  const size_t begin( IsStrictlyLower<MT>::value ? index+1UL : index );
267  const size_t n ( mat_.rows() - begin );
268  return subvector( vec_, begin, n ) * subvector( column( mat_, index ), begin, n );
269  }
270  else if( IsUpper<MT>::value )
271  {
272  const size_t n( IsStrictlyUpper<MT>::value ? index : index+1UL );
273  return subvector( vec_, 0UL, n ) * subvector( column( mat_, index ), 0UL, n );
274  }
275  else
276  {
277  return vec_ * column( mat_, index );
278  }
279  }
280  //**********************************************************************************************
281 
282  //**At function*********************************************************************************
289  inline ReturnType at( size_t index ) const {
290  if( index >= mat_.columns() ) {
291  BLAZE_THROW_OUT_OF_RANGE( "Invalid vector access index" );
292  }
293  return (*this)[index];
294  }
295  //**********************************************************************************************
296 
297  //**Size function*******************************************************************************
302  inline size_t size() const noexcept {
303  return mat_.columns();
304  }
305  //**********************************************************************************************
306 
307  //**Left operand access*************************************************************************
312  inline LeftOperand leftOperand() const noexcept {
313  return vec_;
314  }
315  //**********************************************************************************************
316 
317  //**Right operand access************************************************************************
322  inline RightOperand rightOperand() const noexcept {
323  return mat_;
324  }
325  //**********************************************************************************************
326 
327  //**********************************************************************************************
333  template< typename T >
334  inline bool canAlias( const T* alias ) const noexcept {
335  return vec_.isAliased( alias ) || mat_.isAliased( alias );
336  }
337  //**********************************************************************************************
338 
339  //**********************************************************************************************
345  template< typename T >
346  inline bool isAliased( const T* alias ) const noexcept {
347  return vec_.isAliased( alias ) || mat_.isAliased( alias );
348  }
349  //**********************************************************************************************
350 
351  //**********************************************************************************************
356  inline bool isAligned() const noexcept {
357  return mat_.isAligned();
358  }
359  //**********************************************************************************************
360 
361  //**********************************************************************************************
366  inline bool canSMPAssign() const noexcept {
367  return ( size() > SMP_TSVECDMATMULT_THRESHOLD );
368  }
369  //**********************************************************************************************
370 
371  private:
372  //**Member variables****************************************************************************
373  LeftOperand vec_;
374  RightOperand mat_;
375  //**********************************************************************************************
376 
377  //**Assignment to dense vectors*****************************************************************
390  template< typename VT2 > // Type of the target dense vector
391  friend inline void assign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
392  {
394 
395  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
396 
397  // Evaluation of the left-hand side sparse vector operand
398  LT x( serial( rhs.vec_ ) );
399  if( x.nonZeros() == 0UL ) {
400  reset( ~lhs );
401  return;
402  }
403 
404  // Evaluation of the right-hand side dense matrix operand
405  RT A( serial( rhs.mat_ ) );
406 
407  // Checking the evaluated operands
408  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
409  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
410  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
411  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
412 
413  // Performing the sparse vector-dense matrix multiplication
414  TSVecDMatMultExpr::selectAssignKernel( ~lhs, x, A );
415  }
417  //**********************************************************************************************
418 
419  //**Default assignment to dense vectors*********************************************************
433  template< typename VT1 // Type of the left-hand side target vector
434  , typename VT2 // Type of the left-hand side vector operand
435  , typename MT1 > // Type of the right-hand side matrix operand
437  selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
438  {
440 
441  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
442 
443  const size_t N( A.columns() );
444 
445  ConstIterator element( x.begin() );
446  const ConstIterator end( x.end() );
447 
448  size_t last( 0UL );
449 
450  if( IsUpper<MT1>::value ) {
451  const size_t jend( IsStrictlyUpper<MT1>::value ? element->index()+1UL : element->index() );
452  for( size_t j=0UL; j<jend; ++j )
453  reset( y[j] );
454  }
455 
456  for( ; element!=end; ++element )
457  {
458  const size_t index( element->index() );
459 
460  if( IsDiagonal<MT1>::value )
461  {
462 
463 
464  for( size_t j=last; j<index; ++j )
465  reset( y[j] );
466 
467  y[index] = element->value() * A(index,index);
468  last = index + 1UL;
469  }
470  else
471  {
472  const size_t jbegin( ( IsUpper<MT1>::value )
473  ?( IsStrictlyUpper<MT1>::value ? index+1UL : index )
474  :( 0UL ) );
475  const size_t jend( ( IsLower<MT1>::value )
476  ?( IsStrictlyLower<MT1>::value ? index : index+1UL )
477  :( N ) );
478  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
479 
480  for( size_t j=jbegin; j<last; ++j ) {
481  y[j] += element->value() * A(index,j);
482  }
483  for( size_t j=last; j<jend; ++j ) {
484  y[j] = element->value() * A(index,j);
485  }
486 
487  last = jend;
488  }
489  }
490 
491  if( IsLower<MT1>::value ) {
492  for( size_t j=last; j<N; ++j )
493  reset( y[j] );
494  }
495  }
497  //**********************************************************************************************
498 
499  //**Optimized assignment to dense vectors*******************************************************
513  template< typename VT1 // Type of the left-hand side target vector
514  , typename VT2 // Type of the left-hand side vector operand
515  , typename MT1 > // Type of the right-hand side matrix operand
516  static inline EnableIf_< UseOptimizedKernel<VT1,VT2,MT1> >
517  selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
518  {
519  typedef ConstIterator_< RemoveReference_<LT> > ConstIterator;
520 
521  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
522 
523  const size_t N( A.columns() );
524 
525  ConstIterator element( x.begin() );
526  const ConstIterator end( x.end() );
527 
528  const size_t ipos( x.nonZeros() & size_t(-4) );
529  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
530 
531  if( ipos > 3UL )
532  {
533  const size_t i1( element->index() );
534  const VET v1( element->value() );
535  ++element;
536  const size_t i2( element->index() );
537  const VET v2( element->value() );
538  ++element;
539  const size_t i3( element->index() );
540  const VET v3( element->value() );
541  ++element;
542  const size_t i4( element->index() );
543  const VET v4( element->value() );
544  ++element;
545 
546  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
547 
548  for( size_t j=0UL; j<N; ++j ) {
549  y[j] = v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
550  }
551  }
552  else
553  {
554  const size_t i1( element->index() );
555  const VET v1( element->value() );
556  ++element;
557 
558  for( size_t j=0UL; j<N; ++j ) {
559  y[j] = v1 * A(i1,j);
560  }
561  }
562 
563  for( size_t i=(ipos>3UL)?(4UL):(1UL); (i+4UL)<=ipos; i+=4UL )
564  {
565  const size_t i1( element->index() );
566  const VET v1( element->value() );
567  ++element;
568  const size_t i2( element->index() );
569  const VET v2( element->value() );
570  ++element;
571  const size_t i3( element->index() );
572  const VET v3( element->value() );
573  ++element;
574  const size_t i4( element->index() );
575  const VET v4( element->value() );
576  ++element;
577 
578  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
579 
580  const size_t jbegin( ( IsUpper<MT1>::value )
581  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
582  :( 0UL ) );
583  const size_t jend( ( IsLower<MT1>::value )
584  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
585  :( N ) );
586  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
587 
588  for( size_t j=jbegin; j<jend; ++j ) {
589  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
590  }
591  }
592  for( ; element!=end; ++element )
593  {
594  const size_t i1( element->index() );
595  const VET v1( element->value() );
596 
597  const size_t jbegin( ( IsUpper<MT1>::value )
598  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
599  :( 0UL ) );
600  const size_t jend( ( IsLower<MT1>::value )
601  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
602  :( N ) );
603  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
604 
605  for( size_t j=jbegin; j<jend; ++j ) {
606  y[j] += v1 * A(i1,j);
607  }
608  }
609  }
611  //**********************************************************************************************
612 
613  //**Vectorized assignment to dense vectors******************************************************
627  template< typename VT1 // Type of the left-hand side target vector
628  , typename VT2 // Type of the left-hand side vector operand
629  , typename MT1 > // Type of the right-hand side matrix operand
630  static inline EnableIf_< UseVectorizedKernel<VT1,VT2,MT1> >
631  selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
632  {
633  typedef ConstIterator_< RemoveReference_<LT> > ConstIterator;
634 
635  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
636 
637  const size_t N( A.columns() );
638 
639  const bool remainder( !IsPadded<VT1>::value || !IsPadded<MT1>::value );
640 
641  ConstIterator element( x.begin() );
642  const ConstIterator end( x.end() );
643 
644  const size_t ipos( x.nonZeros() & size_t(-4) );
645  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
646 
647  if( ipos > 3UL )
648  {
649  const size_t i1( element->index() );
650  const VET v1( element->value() );
651  ++element;
652  const size_t i2( element->index() );
653  const VET v2( element->value() );
654  ++element;
655  const size_t i3( element->index() );
656  const VET v3( element->value() );
657  ++element;
658  const size_t i4( element->index() );
659  const VET v4( element->value() );
660  ++element;
661 
662  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
663 
664  const SIMDType xmm1( set( v1 ) );
665  const SIMDType xmm2( set( v2 ) );
666  const SIMDType xmm3( set( v3 ) );
667  const SIMDType xmm4( set( v4 ) );
668 
669  const size_t jpos( remainder ? ( N & size_t(-SIMDSIZE) ) : N );
670  BLAZE_INTERNAL_ASSERT( !remainder || ( N - ( N % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
671 
672  size_t j( 0UL );
673 
674  for( ; j<jpos; j+=SIMDSIZE ) {
675  y.store( j, xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
676  }
677  for( ; remainder && j<N; ++j ) {
678  y[j] = v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
679  }
680  }
681  else
682  {
683  const size_t i1( element->index() );
684  const VET v1( element->value() );
685  ++element;
686 
687  const SIMDType xmm1( set( v1 ) );
688 
689  const size_t jpos( remainder ? ( N & size_t(-SIMDSIZE) ) : N );
690  BLAZE_INTERNAL_ASSERT( !remainder || ( N - ( N % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
691 
692  size_t j( 0UL );
693 
694  for( ; j<jpos; j+=SIMDSIZE ) {
695  y.store( j, xmm1 * A.load(i1,j) );
696  }
697  for( ; remainder && j<N; ++j ) {
698  y[j] = v1 * A(i1,j);
699  }
700  }
701 
702  for( size_t i=(ipos>3UL)?(4UL):(1UL); (i+4UL)<=ipos; i+=4UL )
703  {
704  const size_t i1( element->index() );
705  const VET v1( element->value() );
706  ++element;
707  const size_t i2( element->index() );
708  const VET v2( element->value() );
709  ++element;
710  const size_t i3( element->index() );
711  const VET v3( element->value() );
712  ++element;
713  const size_t i4( element->index() );
714  const VET v4( element->value() );
715  ++element;
716 
717  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
718 
719  const SIMDType xmm1( set( v1 ) );
720  const SIMDType xmm2( set( v2 ) );
721  const SIMDType xmm3( set( v3 ) );
722  const SIMDType xmm4( set( v4 ) );
723 
724  const size_t jbegin( ( IsUpper<MT1>::value )
725  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
726  :( 0UL ) );
727  const size_t jend( ( IsLower<MT1>::value )
728  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
729  :( N ) );
730  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
731 
732  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
733  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
734 
735  size_t j( jbegin );
736 
737  for( ; j<jpos; j+=SIMDSIZE ) {
738  y.store( j, y.load(j) + xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
739  }
740  for( ; remainder && j<jend; ++j ) {
741  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
742  }
743  }
744  for( ; element!=end; ++element )
745  {
746  const size_t i1( element->index() );
747  const VET v1( element->value() );
748 
749  const SIMDType xmm1( set( v1 ) );
750 
751  const size_t jbegin( ( IsUpper<MT1>::value )
752  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
753  :( 0UL ) );
754  const size_t jend( ( IsLower<MT1>::value )
755  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
756  :( N ) );
757  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
758 
759  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
760  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
761 
762  size_t j( jbegin );
763 
764  for( ; j<jpos; j+=SIMDSIZE ) {
765  y.store( j, y.load(j) + xmm1 * A.load(i1,j) );
766  }
767  for( ; remainder && j<jend; ++j ) {
768  y[j] += v1 * A(i1,j);
769  }
770  }
771  }
773  //**********************************************************************************************
774 
775  //**Assignment to sparse vectors****************************************************************
788  template< typename VT2 > // Type of the target sparse vector
789  friend inline void assign( SparseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
790  {
792 
795  BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE( CompositeType_<ResultType> );
796 
797  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
798 
799  const ResultType tmp( serial( rhs ) );
800  assign( ~lhs, tmp );
801  }
803  //**********************************************************************************************
804 
805  //**Addition assignment to dense vectors********************************************************
817  template< typename VT2 > // Type of the target dense vector
818  friend inline void addAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
819  {
821 
822  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
823 
824  // Evaluation of the left-hand side sparse vector operand
825  LT x( serial( rhs.vec_ ) );
826  if( x.nonZeros() == 0UL ) return;
827 
828  // Evaluation of the right-hand side dense matrix operand
829  RT A( serial( rhs.mat_ ) );
830 
831  // Checking the evaluated operands
832  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
833  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
834  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
835  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
836 
837  // Performing the sparse vector-dense matrix multiplication
838  TSVecDMatMultExpr::selectAddAssignKernel( ~lhs, x, A );
839  }
840  //**********************************************************************************************
841 
842  //**Default addition assignment to dense vectors************************************************
856  template< typename VT1 // Type of the left-hand side target vector
857  , typename VT2 // Type of the left-hand side vector operand
858  , typename MT1 > // Type of the right-hand side matrix operand
860  selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
861  {
863 
864  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
865 
866  const size_t N( A.columns() );
867 
868  ConstIterator element( x.begin() );
869  const ConstIterator end( x.end() );
870 
871  for( ; element!=end; ++element )
872  {
873  const size_t index( element->index() );
874 
876  {
877  y[index] += A(index,index) * element->value();
878  }
879  else
880  {
881  const size_t jbegin( ( IsUpper<MT1>::value )
882  ?( IsStrictlyUpper<MT1>::value ? index+1UL : index )
883  :( 0UL ) );
884  const size_t jend( ( IsLower<MT1>::value )
885  ?( IsStrictlyLower<MT1>::value ? index : index+1UL )
886  :( N ) );
887  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
888 
889  for( size_t j=jbegin; j<jend; ++j ) {
890  y[j] += element->value() * A(index,j);
891  }
892  }
893  }
894  }
896  //**********************************************************************************************
897 
898  //**Optimized addition assignment to dense vectors**********************************************
912  template< typename VT1 // Type of the left-hand side target vector
913  , typename VT2 // Type of the left-hand side vector operand
914  , typename MT1 > // Type of the right-hand side matrix operand
915  static inline EnableIf_< UseOptimizedKernel<VT1,VT2,MT1> >
916  selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
917  {
918  typedef ConstIterator_< RemoveReference_<LT> > ConstIterator;
919 
920  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
921 
922  const size_t N( A.columns() );
923 
924  ConstIterator element( x.begin() );
925  const ConstIterator end( x.end() );
926 
927  const size_t ipos( x.nonZeros() & size_t(-4) );
928  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
929 
930  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
931  {
932  const size_t i1( element->index() );
933  const VET v1( element->value() );
934  ++element;
935  const size_t i2( element->index() );
936  const VET v2( element->value() );
937  ++element;
938  const size_t i3( element->index() );
939  const VET v3( element->value() );
940  ++element;
941  const size_t i4( element->index() );
942  const VET v4( element->value() );
943  ++element;
944 
945  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
946 
947  const size_t jbegin( ( IsUpper<MT1>::value )
948  ?( IsStrictlyUpper<MT1>::value ? i+1UL : i1 )
949  :( 0UL ) );
950  const size_t jend( ( IsLower<MT1>::value )
951  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
952  :( N ) );
953  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
954 
955  for( size_t j=jbegin; j<jend; ++j ) {
956  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
957  }
958  }
959  for( ; element!=end; ++element )
960  {
961  const size_t i1( element->index() );
962  const VET v1( element->value() );
963 
964  const size_t jbegin( ( IsUpper<MT1>::value )
965  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
966  :( 0UL ) );
967  const size_t jend( ( IsLower<MT1>::value )
968  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
969  :( N ) );
970  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
971 
972  for( size_t j=jbegin; j<jend; ++j ) {
973  y[j] += v1 * A(i1,j);
974  }
975  }
976  }
978  //**********************************************************************************************
979 
980  //**Vectorized addition assignment to dense vectors*********************************************
994  template< typename VT1 // Type of the left-hand side target vector
995  , typename VT2 // Type of the left-hand side vector operand
996  , typename MT1 > // Type of the right-hand side matrix operand
997  static inline EnableIf_< UseVectorizedKernel<VT1,VT2,MT1> >
998  selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
999  {
1000  typedef ConstIterator_< RemoveReference_<LT> > ConstIterator;
1001 
1002  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1003 
1004  const size_t N( A.columns() );
1005 
1006  const bool remainder( !IsPadded<VT1>::value || !IsPadded<MT1>::value );
1007 
1008  ConstIterator element( x.begin() );
1009  const ConstIterator end( x.end() );
1010 
1011  const size_t ipos( x.nonZeros() & size_t(-4) );
1012  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1013 
1014  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1015  {
1016  const size_t i1( element->index() );
1017  const VET v1( element->value() );
1018  ++element;
1019  const size_t i2( element->index() );
1020  const VET v2( element->value() );
1021  ++element;
1022  const size_t i3( element->index() );
1023  const VET v3( element->value() );
1024  ++element;
1025  const size_t i4( element->index() );
1026  const VET v4( element->value() );
1027  ++element;
1028 
1029  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1030 
1031  const SIMDType xmm1( set( v1 ) );
1032  const SIMDType xmm2( set( v2 ) );
1033  const SIMDType xmm3( set( v3 ) );
1034  const SIMDType xmm4( set( v4 ) );
1035 
1036  const size_t jbegin( ( IsUpper<MT1>::value )
1037  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1038  :( 0UL ) );
1039  const size_t jend( ( IsLower<MT1>::value )
1040  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
1041  :( N ) );
1042  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1043 
1044  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1045  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1046 
1047  size_t j( jbegin );
1048 
1049  for( ; j<jpos; j+=SIMDSIZE ) {
1050  y.store( j, y.load(j) + xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
1051  }
1052  for( ; remainder && j<jend; ++j ) {
1053  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1054  }
1055  }
1056  for( ; element!=end; ++element )
1057  {
1058  const size_t i1( element->index() );
1059  const VET v1( element->value() );
1060 
1061  const SIMDType xmm1( set( v1 ) );
1062 
1063  const size_t jbegin( ( IsUpper<MT1>::value )
1064  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1065  :( 0UL ) );
1066  const size_t jend( ( IsLower<MT1>::value )
1067  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
1068  :( N ) );
1069  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1070 
1071  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1072  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1073 
1074  size_t j( jbegin );
1075 
1076  for( ; j<jpos; j+=SIMDSIZE ) {
1077  y.store( j, y.load(j) + xmm1 * A.load(i1,j) );
1078  }
1079  for( ; remainder && j<jend; ++j ) {
1080  y[j] += v1 * A(i1,j);
1081  }
1082  }
1083  }
1085  //**********************************************************************************************
1086 
1087  //**Addition assignment to sparse vectors*******************************************************
1088  // No special implementation for the addition assignment to sparse vectors.
1089  //**********************************************************************************************
1090 
1091  //**Subtraction assignment to dense vectors*****************************************************
1103  template< typename VT2 > // Type of the target dense vector
1104  friend inline void subAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1105  {
1107 
1108  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1109 
1110  // Evaluation of the left-hand side sparse vector operand
1111  LT x( serial( rhs.vec_ ) );
1112  if( x.nonZeros() == 0UL ) return;
1113 
1114  // Evaluation of the right-hand side dense matrix operand
1115  RT A( serial( rhs.mat_ ) );
1116 
1117  // Checking the evaluated operands
1118  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1119  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1120  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1121  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1122 
1123  // Performing the sparse vector-dense matrix multiplication
1124  TSVecDMatMultExpr::selectSubAssignKernel( ~lhs, x, A );
1125  }
1126  //**********************************************************************************************
1127 
1128  //**Default subtraction assignment to dense vectors*********************************************
1142  template< typename VT1 // Type of the left-hand side target vector
1143  , typename VT2 // Type of the left-hand side vector operand
1144  , typename MT1 > // Type of the right-hand side matrix operand
1146  selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1147  {
1149 
1150  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1151 
1152  const size_t N( A.columns() );
1153 
1154  ConstIterator element( x.begin() );
1155  const ConstIterator end( x.end() );
1156 
1157  for( ; element!=end; ++element )
1158  {
1159  const size_t index( element->index() );
1160 
1162  {
1163  y[index] -= A(index,index) * element->value();
1164  }
1165  else
1166  {
1167  const size_t jbegin( ( IsUpper<MT1>::value )
1168  ?( IsStrictlyUpper<MT1>::value ? index+1UL : index )
1169  :( 0UL ) );
1170  const size_t jend( ( IsLower<MT1>::value )
1171  ?( IsStrictlyLower<MT1>::value ? index : index+1UL )
1172  :( N ) );
1173  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1174 
1175  for( size_t j=jbegin; j<jend; ++j ) {
1176  y[j] -= element->value() * A(index,j);
1177  }
1178  }
1179  }
1180  }
1182  //**********************************************************************************************
1183 
1184  //**Optimized subtraction assignment to dense vectors*******************************************
1198  template< typename VT1 // Type of the left-hand side target vector
1199  , typename VT2 // Type of the left-hand side vector operand
1200  , typename MT1 > // Type of the right-hand side matrix operand
1201  static inline EnableIf_< UseOptimizedKernel<VT1,VT2,MT1> >
1202  selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1203  {
1204  typedef ConstIterator_< RemoveReference_<LT> > ConstIterator;
1205 
1206  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1207 
1208  const size_t N( A.columns() );
1209 
1210  ConstIterator element( x.begin() );
1211  const ConstIterator end( x.end() );
1212 
1213  const size_t ipos( x.nonZeros() & size_t(-4) );
1214  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1215 
1216  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1217  {
1218  const size_t i1( element->index() );
1219  const VET v1( element->value() );
1220  ++element;
1221  const size_t i2( element->index() );
1222  const VET v2( element->value() );
1223  ++element;
1224  const size_t i3( element->index() );
1225  const VET v3( element->value() );
1226  ++element;
1227  const size_t i4( element->index() );
1228  const VET v4( element->value() );
1229  ++element;
1230 
1231  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1232 
1233  const size_t jbegin( ( IsUpper<MT1>::value )
1234  ?( IsStrictlyUpper<MT1>::value ? i+1UL : i1 )
1235  :( 0UL ) );
1236  const size_t jend( ( IsLower<MT1>::value )
1237  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
1238  :( N ) );
1239  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1240 
1241  for( size_t j=jbegin; j<jend; ++j ) {
1242  y[j] -= v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1243  }
1244  }
1245  for( ; element!=end; ++element )
1246  {
1247  const size_t i1( element->index() );
1248  const VET v1( element->value() );
1249 
1250  const size_t jbegin( ( IsUpper<MT1>::value )
1251  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
1252  :( 0UL ) );
1253  const size_t jend( ( IsLower<MT1>::value )
1254  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
1255  :( N ) );
1256  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1257 
1258  for( size_t j=jbegin; j<jend; ++j ) {
1259  y[j] -= v1 * A(i1,j);
1260  }
1261  }
1262  }
1264  //**********************************************************************************************
1265 
1266  //**Vectorized subtraction assignment to dense vectors******************************************
1280  template< typename VT1 // Type of the left-hand side target vector
1281  , typename VT2 // Type of the left-hand side vector operand
1282  , typename MT1 > // Type of the right-hand side matrix operand
1283  static inline EnableIf_< UseVectorizedKernel<VT1,VT2,MT1> >
1284  selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1285  {
1286  typedef ConstIterator_< RemoveReference_<LT> > ConstIterator;
1287 
1288  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1289 
1290  const size_t N( A.columns() );
1291 
1292  const bool remainder( !IsPadded<VT1>::value || !IsPadded<MT1>::value );
1293 
1294  ConstIterator element( x.begin() );
1295  const ConstIterator end( x.end() );
1296 
1297  const size_t ipos( x.nonZeros() & size_t(-4) );
1298  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1299 
1300  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1301  {
1302  const size_t i1( element->index() );
1303  const VET v1( element->value() );
1304  ++element;
1305  const size_t i2( element->index() );
1306  const VET v2( element->value() );
1307  ++element;
1308  const size_t i3( element->index() );
1309  const VET v3( element->value() );
1310  ++element;
1311  const size_t i4( element->index() );
1312  const VET v4( element->value() );
1313  ++element;
1314 
1315  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1316 
1317  const SIMDType xmm1( set( v1 ) );
1318  const SIMDType xmm2( set( v2 ) );
1319  const SIMDType xmm3( set( v3 ) );
1320  const SIMDType xmm4( set( v4 ) );
1321 
1322  const size_t jbegin( ( IsUpper<MT1>::value )
1323  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1324  :( 0UL ) );
1325  const size_t jend( ( IsLower<MT1>::value )
1326  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
1327  :( N ) );
1328  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1329 
1330  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1331  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1332 
1333  size_t j( jbegin );
1334 
1335  for( ; j<jpos; j+=SIMDSIZE ) {
1336  y.store( j, y.load(j) - xmm1 * A.load(i1,j) - xmm2 * A.load(i2,j) - xmm3 * A.load(i3,j) - xmm4 * A.load(i4,j) );
1337  }
1338  for( ; remainder && j<jend; ++j ) {
1339  y[j] -= v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1340  }
1341  }
1342  for( ; element!=x.end(); ++element )
1343  {
1344  const size_t i1( element->index() );
1345  const VET v1( element->value() );
1346 
1347  const SIMDType xmm1( set( v1 ) );
1348 
1349  const size_t jbegin( ( IsUpper<MT1>::value )
1350  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1351  :( 0UL ) );
1352  const size_t jend( ( IsLower<MT1>::value )
1353  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
1354  :( N ) );
1355  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1356 
1357  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1358  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1359 
1360  size_t j( jbegin );
1361 
1362  for( ; j<jpos; j+=SIMDSIZE ) {
1363  y.store( j, y.load(j) - xmm1 * A.load(i1,j) );
1364  }
1365  for( ; remainder && j<jend; ++j ) {
1366  y[j] -= v1 * A(i1,j);
1367  }
1368  }
1369  }
1371  //**********************************************************************************************
1372 
1373  //**Subtraction assignment to sparse vectors****************************************************
1374  // No special implementation for the subtraction assignment to sparse vectors.
1375  //**********************************************************************************************
1376 
1377  //**Multiplication assignment to dense vectors**************************************************
1389  template< typename VT2 > // Type of the target dense vector
1390  friend inline void multAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1391  {
1393 
1397 
1398  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1399 
1400  const ResultType tmp( serial( rhs ) );
1401  multAssign( ~lhs, tmp );
1402  }
1403  //**********************************************************************************************
1404 
1405  //**Multiplication assignment to sparse vectors*************************************************
1406  // No special implementation for the multiplication assignment to sparse vectors.
1407  //**********************************************************************************************
1408 
1409  //**Division assignment to dense vectors********************************************************
1421  template< typename VT2 > // Type of the target dense vector
1422  friend inline void divAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1423  {
1425 
1429 
1430  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1431 
1432  const ResultType tmp( serial( rhs ) );
1433  divAssign( ~lhs, tmp );
1434  }
1435  //**********************************************************************************************
1436 
1437  //**Division assignment to sparse vectors*******************************************************
1438  // No special implementation for the division assignment to sparse vectors.
1439  //**********************************************************************************************
1440 
1441  //**SMP assignment to dense vectors*************************************************************
1456  template< typename VT2 > // Type of the target dense vector
1457  friend inline EnableIf_< UseSMPAssign<VT2> >
1459  {
1461 
1462  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1463 
1464  // Evaluation of the left-hand side sparse vector operand
1465  LT x( rhs.vec_ );
1466  if( x.nonZeros() == 0UL ) {
1467  reset( ~lhs );
1468  return;
1469  }
1470 
1471  // Evaluation of the right-hand side dense matrix operand
1472  RT A( rhs.mat_ );
1473 
1474  // Checking the evaluated operands
1475  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1476  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1477  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1478  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1479 
1480  // Performing the sparse vector-dense matrix multiplication
1481  smpAssign( ~lhs, x * A );
1482  }
1484  //**********************************************************************************************
1485 
1486  //**SMP assignment to sparse vectors************************************************************
1501  template< typename VT2 > // Type of the target sparse vector
1502  friend inline EnableIf_< UseSMPAssign<VT2> >
1503  smpAssign( SparseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1504  {
1506 
1509  BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE( CompositeType_<ResultType> );
1510 
1511  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1512 
1513  const ResultType tmp( rhs );
1514  smpAssign( ~lhs, tmp );
1515  }
1517  //**********************************************************************************************
1518 
1519  //**SMP addition assignment to dense vectors****************************************************
1533  template< typename VT2 > // Type of the target dense vector
1534  friend inline EnableIf_< UseSMPAssign<VT2> >
1536  {
1538 
1539  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1540 
1541  // Evaluation of the left-hand side sparse vector operand
1542  LT x( rhs.vec_ );
1543  if( x.nonZeros() == 0UL ) return;
1544 
1545  // Evaluation of the right-hand side dense matrix operand
1546  RT A( rhs.mat_ );
1547 
1548  // Checking the evaluated operands
1549  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1550  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1551  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1552  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1553 
1554  // Performing the sparse vector-dense matrix multiplication
1555  smpAddAssign( ~lhs, x * A );
1556  }
1557  //**********************************************************************************************
1558 
1559  //**SMP addition assignment to sparse vectors***************************************************
1560  // No special implementation for the SMP addition assignment to sparse vectors.
1561  //**********************************************************************************************
1562 
1563  //**SMP subtraction assignment to dense vectors*************************************************
1577  template< typename VT2 > // Type of the target dense vector
1578  friend inline EnableIf_< UseSMPAssign<VT2> >
1580  {
1582 
1583  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1584 
1585  // Evaluation of the left-hand side sparse vector operand
1586  LT x( rhs.vec_ );
1587  if( x.nonZeros() == 0UL ) return;
1588 
1589  // Evaluation of the right-hand side dense matrix operand
1590  RT A( rhs.mat_ );
1591 
1592  // Checking the evaluated operands
1593  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1594  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1595  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1596  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1597 
1598  // Performing the sparse vector-dense matrix multiplication
1599  smpSubAssign( ~lhs, x * A );
1600  }
1601  //**********************************************************************************************
1602 
1603  //**SMP subtraction assignment to sparse vectors************************************************
1604  // No special implementation for the SMP subtraction assignment to sparse vectors.
1605  //**********************************************************************************************
1606 
1607  //**SMP multiplication assignment to dense vectors**********************************************
1621  template< typename VT2 > // Type of the target dense vector
1622  friend inline EnableIf_< UseSMPAssign<VT2> >
1624  {
1626 
1630 
1631  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1632 
1633  const ResultType tmp( rhs );
1634  smpMultAssign( ~lhs, tmp );
1635  }
1636  //**********************************************************************************************
1637 
1638  //**SMP multiplication assignment to sparse vectors*********************************************
1639  // No special implementation for the SMP multiplication assignment to sparse vectors.
1640  //**********************************************************************************************
1641 
1642  //**SMP division assignment to dense vectors****************************************************
1656  template< typename VT2 > // Type of the target dense vector
1657  friend inline EnableIf_< UseSMPAssign<VT2> >
1659  {
1661 
1665 
1666  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1667 
1668  const ResultType tmp( rhs );
1669  smpDivAssign( ~lhs, tmp );
1670  }
1671  //**********************************************************************************************
1672 
1673  //**SMP division assignment to sparse vectors***************************************************
1674  // No special implementation for the SMP division assignment to sparse vectors.
1675  //**********************************************************************************************
1676 
1677  //**Compile time checks*************************************************************************
1685  //**********************************************************************************************
1686 };
1687 //*************************************************************************************************
1688 
1689 
1690 
1691 
1692 //=================================================================================================
1693 //
1694 // GLOBAL BINARY ARITHMETIC OPERATORS
1695 //
1696 //=================================================================================================
1697 
1698 //*************************************************************************************************
1729 template< typename T1, typename T2 >
1730 inline const DisableIf_< IsMatMatMultExpr<T2>, TSVecDMatMultExpr<T1,T2> >
1732 {
1734 
1735  if( (~vec).size() != (~mat).rows() ) {
1736  BLAZE_THROW_INVALID_ARGUMENT( "Vector and matrix sizes do not match" );
1737  }
1738 
1739  return TSVecDMatMultExpr<T1,T2>( ~vec, ~mat );
1740 }
1741 //*************************************************************************************************
1742 
1743 
1744 
1745 
1746 //=================================================================================================
1747 //
1748 // GLOBAL RESTRUCTURING BINARY ARITHMETIC OPERATORS
1749 //
1750 //=================================================================================================
1751 
1752 //*************************************************************************************************
1765 template< typename T1 // Type of the left-hand side sparse vector
1766  , typename T2 // Type of the right-hand side dense matrix
1767  , bool SO > // Storage order of the right-hand side dense matrix
1768 inline const EnableIf_< IsMatMatMultExpr<T2>, MultExprTrait_<T1,T2> >
1770 {
1772 
1774 
1775  return ( vec * (~mat).leftOperand() ) * (~mat).rightOperand();
1776 }
1777 //*************************************************************************************************
1778 
1779 
1780 
1781 
1782 //=================================================================================================
1783 //
1784 // SIZE SPECIALIZATIONS
1785 //
1786 //=================================================================================================
1787 
1788 //*************************************************************************************************
1790 template< typename VT, typename MT >
1791 struct Size< TSVecDMatMultExpr<VT,MT> > : public Columns<MT>
1792 {};
1794 //*************************************************************************************************
1795 
1796 
1797 
1798 
1799 //=================================================================================================
1800 //
1801 // ISALIGNED SPECIALIZATIONS
1802 //
1803 //=================================================================================================
1804 
1805 //*************************************************************************************************
1807 template< typename VT, typename MT >
1808 struct IsAligned< TSVecDMatMultExpr<VT,MT> >
1809  : public BoolConstant< IsAligned<MT>::value >
1810 {};
1812 //*************************************************************************************************
1813 
1814 
1815 
1816 
1817 //=================================================================================================
1818 //
1819 // EXPRESSION TRAIT SPECIALIZATIONS
1820 //
1821 //=================================================================================================
1822 
1823 //*************************************************************************************************
1825 template< typename VT, typename MT, bool AF >
1826 struct SubvectorExprTrait< TSVecDMatMultExpr<VT,MT>, AF >
1827 {
1828  public:
1829  //**********************************************************************************************
1830  using Type = MultExprTrait_< SubvectorExprTrait_<const VT,AF>
1831  , SubmatrixExprTrait_<const MT,AF> >;
1832  //**********************************************************************************************
1833 };
1835 //*************************************************************************************************
1836 
1837 } // namespace blaze
1838 
1839 #endif
MultTrait_< VRT, MRT > ResultType
Result type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:200
friend EnableIf_< UseSMPAssign< VT2 > > smpDivAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP division assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ...
Definition: TSVecDMatMultExpr.h:1658
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
friend EnableIf_< UseSMPAssign< VT2 > > smpAddAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP addition assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ...
Definition: TSVecDMatMultExpr.h:1535
Header file for auxiliary alias declarations.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:72
constexpr bool useOptimizedKernels
Configuration switch for optimized kernels.This configuration switch enables/disables all optimized c...
Definition: Optimizations.h:84
ElementType_< ResultType > ElementType
Resulting element type.
Definition: TSVecDMatMultExpr.h:202
CompositeType_< VT > VCT
Composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:120
friend void multAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Multiplication assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ...
Definition: TSVecDMatMultExpr.h:1390
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:7800
size_t size() const noexcept
Returns the current size/dimension of the vector.
Definition: TSVecDMatMultExpr.h:302
Header file for basic type definitions.
TSVecDMatMultExpr< VT, MT > This
Type of this TSVecDMatMultExpr instance.
Definition: TSVecDMatMultExpr.h:199
Expression object for transpose sparse vector-dense matrix multiplications.The TSVecDMatMultExpr clas...
Definition: Forward.h:150
RightOperand rightOperand() const noexcept
Returns the right-hand side dense matrix operand.
Definition: TSVecDMatMultExpr.h:322
friend EnableIf_< UseSMPAssign< VT2 > > smpMultAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP multiplication assignment of a transpose sparse vector-dense matrix multiplication to a dense vec...
Definition: TSVecDMatMultExpr.h:1623
Header file for the serial shim.
Header file for the IsDiagonal type trait.
BLAZE_ALWAYS_INLINE size_t size(const Vector< VT, TF > &vector) noexcept
Returns the current size/dimension of the vector.
Definition: Vector.h:258
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the IsSame and IsStrictlySame type traits.
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:188
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:162
LeftOperand vec_
Left-hand side sparse vector of the multiplication expression.
Definition: TSVecDMatMultExpr.h:373
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:315
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:533
IfTrue_< evaluateVector, const VRT, VCT > LT
Type for the assignment of the left-hand side sparse vector operand.
Definition: TSVecDMatMultExpr.h:214
Header file for the DenseVector base class.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:88
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:162
ReturnType at(size_t index) const
Checked access to the vector elements.
Definition: TSVecDMatMultExpr.h:289
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:201
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:723
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:245
Header file for the Computation base class.
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TSVecDMatMultExpr.h:366
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:88
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:323
DisableIf_< Or< IsComputation< MT >, IsTransExpr< MT > >, ColumnExprTrait_< MT > > column(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific column of the given matrix.
Definition: Column.h:126
If_< IsExpression< VT >, const VT, const VT & > LeftOperand
Composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:208
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
Constraint on the data type.
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
LeftOperand leftOperand() const noexcept
Returns the left-hand side sparse vector operand.
Definition: TSVecDMatMultExpr.h:312
Constraint on the data type.
Header file for the MultExprTrait class template.
RightOperand mat_
Right-hand side dense matrix of the multiplication expression.
Definition: TSVecDMatMultExpr.h:374
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:72
SubvectorExprTrait_< VT, unaligned > subvector(Vector< VT, TF > &vector, size_t index, size_t size)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:152
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
CompositeType_< MT > MCT
Composite type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:121
Header file for the DisableIf class template.
friend void addAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Addition assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:818
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TSVecDMatMultExpr.h:334
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
Header file for the If class template.
friend void divAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Division assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:1422
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2647
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
ElementType_< VRT > VET
Element type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:118
Header file for the IsMatMatMultExpr type trait class.
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
Header file for the Columns type trait.
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
Base class for N-dimensional dense vectors.The DenseVector class is a base class for all arbitrarily ...
Definition: DenseVector.h:70
Header file for the IsLower type trait.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional vector type...
Definition: SparseVector.h:61
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:90
IfTrue_< evaluateMatrix, const MRT, MCT > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: TSVecDMatMultExpr.h:217
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:60
Constraint on the data type.
Header file for the exception macros of the math module.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:254
Header file for all forward declarations for expression class templates.
Constraint on the data type.
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TSVecDMatMultExpr.h:356
If_< IsExpression< MT >, const MT, const MT & > RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TSVecDMatMultExpr.h:211
BLAZE_ALWAYS_INLINE const EnableIf_< And< IsIntegral< T >, HasSize< T, 1UL > >, If_< IsSigned< T >, SIMDint8, SIMDuint8 > > set(T value) noexcept
Sets all values in the vector to the given 1-byte integral value.
Definition: Set.h:76
ResultType_< VT > VRT
Result type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:116
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type, a compilation error is created.
Definition: Symmetric.h:79
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
Header file for run time assertion macros.
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:204
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:160
Header file for the reset shim.
Constraint on the data type.
ReturnType operator[](size_t index) const
Subscript operator for the direct access to the vector elements.
Definition: TSVecDMatMultExpr.h:257
Header file for the TVecMatMultExpr base class.
Constraint on the data type.
TSVecDMatMultExpr(const VT &vec, const MT &mat) noexcept
Constructor for the TSVecDMatMultExpr class.
Definition: TSVecDMatMultExpr.h:243
IntegralConstant< bool, B > BoolConstant
Generic wrapper for a compile time constant boolean value.The BoolConstant class template represents ...
Definition: IntegralConstant.h:100
Header file for the RemoveReference type trait.
ResultType_< MT > MRT
Result type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:117
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:223
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
BLAZE_ALWAYS_INLINE size_t rows(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of rows of the matrix.
Definition: Matrix.h:314
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional vector type...
Definition: DenseVector.h:61
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_TVECMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid vector/matrix ...
Definition: TVecMatMultExpr.h:110
Header file for the AreSIMDCombinable type trait.
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TSVecDMatMultExpr.h:346
Header file for the IsComputation type trait class.
Base class for sparse vectors.The SparseVector class is a base class for all arbitrarily sized (N-dim...
Definition: Forward.h:110
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
Header file for the IntegralConstant class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a row dense or sparse vector type (i...
Definition: RowVector.h:61
Header file for the SubvectorExprTrait class template.
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: TSVecDMatMultExpr.h:203
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:403
friend void subAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Subtraction assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:1104
Header file for the IsUpper type trait.
friend EnableIf_< UseSMPAssign< VT2 > > smpSubAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP subtraction assignment of a transpose sparse vector-dense matrix multiplication to a dense vector...
Definition: TSVecDMatMultExpr.h:1579
Header file for the IsResizable type trait.
const ResultType CompositeType
Data type for composite expression templates.
Definition: TSVecDMatMultExpr.h:205
ElementType_< MRT > MET
Element type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:119
Header file for the Size type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Constraint on the transpose flag of vector types.
Header file for the IsExpression type trait class.
Header file for the FunctionTrace class.