Blaze  3.6
TSVecDMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TSVECDMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TSVECDMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
53 #include <blaze/math/Exception.h>
59 #include <blaze/math/shims/Reset.h>
61 #include <blaze/math/SIMD.h>
78 #include <blaze/math/views/Check.h>
81 #include <blaze/util/Assert.h>
82 #include <blaze/util/DisableIf.h>
83 #include <blaze/util/EnableIf.h>
85 #include <blaze/util/MaybeUnused.h>
86 #include <blaze/util/mpl/If.h>
87 #include <blaze/util/Types.h>
88 
89 
90 namespace blaze {
91 
92 //=================================================================================================
93 //
94 // CLASS TSVECDMATMULTEXPR
95 //
96 //=================================================================================================
97 
98 //*************************************************************************************************
105 template< typename VT // Type of the left-hand side sparse vector
106  , typename MT > // Type of the right-hand side dense matrix
107 class TSVecDMatMultExpr
108  : public TVecMatMultExpr< DenseVector< TSVecDMatMultExpr<VT,MT>, true > >
109  , private Computation
110 {
111  private:
112  //**Type definitions****************************************************************************
119  //**********************************************************************************************
120 
121  //**********************************************************************************************
123  static constexpr bool evaluateVector = ( IsComputation_v<VT> || RequiresEvaluation_v<VT> );
124  //**********************************************************************************************
125 
126  //**********************************************************************************************
128  static constexpr bool evaluateMatrix = RequiresEvaluation_v<MT>;
129  //**********************************************************************************************
130 
131  //**********************************************************************************************
133 
137  template< typename T1 >
138  static constexpr bool UseSMPAssign_v = ( evaluateVector || evaluateMatrix );
140  //**********************************************************************************************
141 
142  //**********************************************************************************************
144 
148  template< typename T1, typename T2, typename T3 >
149  static constexpr bool UseVectorizedKernel_v =
150  ( useOptimizedKernels &&
151  !IsDiagonal_v<T3> &&
152  T1::simdEnabled && T3::simdEnabled &&
153  IsSIMDCombinable_v< ElementType_t<T1>
155  , ElementType_t<T3> > &&
156  HasSIMDAdd_v< ElementType_t<T2>, ElementType_t<T3> > &&
157  HasSIMDMult_v< ElementType_t<T2>, ElementType_t<T3> > );
159  //**********************************************************************************************
160 
161  //**********************************************************************************************
163 
167  template< typename T1, typename T2, typename T3 >
168  static constexpr bool UseOptimizedKernel_v =
169  ( useOptimizedKernels &&
170  !UseVectorizedKernel_v<T1,T2,T3> &&
171  !IsDiagonal_v<T3> &&
172  !IsResizable_v< ElementType_t<T1> > &&
173  !IsResizable_v<VET> );
175  //**********************************************************************************************
176 
177  //**********************************************************************************************
179 
182  template< typename T1, typename T2, typename T3 >
183  static constexpr bool UseDefaultKernel_v =
184  ( !UseVectorizedKernel_v<T1,T2,T3> && !UseOptimizedKernel_v<T1,T2,T3> );
186  //**********************************************************************************************
187 
188  public:
189  //**Type definitions****************************************************************************
196  using ReturnType = const ElementType;
197  using CompositeType = const ResultType;
198 
200  using LeftOperand = If_t< IsExpression_v<VT>, const VT, const VT& >;
201 
203  using RightOperand = If_t< IsExpression_v<MT>, const MT, const MT& >;
204 
207 
210  //**********************************************************************************************
211 
212  //**Compilation flags***************************************************************************
214  static constexpr bool simdEnabled =
215  ( !IsDiagonal_v<MT> &&
216  MT::simdEnabled &&
217  HasSIMDAdd_v<VET,MET> &&
218  HasSIMDMult_v<VET,MET> );
219 
221  static constexpr bool smpAssignable =
222  ( !evaluateVector && VT::smpAssignable && !evaluateMatrix && MT::smpAssignable );
223  //**********************************************************************************************
224 
225  //**SIMD properties*****************************************************************************
227  static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
228  //**********************************************************************************************
229 
230  //**Constructor*********************************************************************************
236  explicit inline TSVecDMatMultExpr( const VT& vec, const MT& mat ) noexcept
237  : vec_( vec ) // Left-hand side sparse vector of the multiplication expression
238  , mat_( mat ) // Right-hand side dense matrix of the multiplication expression
239  {
240  BLAZE_INTERNAL_ASSERT( vec_.size() == mat_.rows(), "Invalid vector and matrix sizes" );
241  }
242  //**********************************************************************************************
243 
244  //**Subscript operator**************************************************************************
250  inline ReturnType operator[]( size_t index ) const {
251  BLAZE_INTERNAL_ASSERT( index < mat_.columns(), "Invalid vector access index" );
252 
253  if( IsDiagonal_v<MT> )
254  {
255  return vec_[index] * mat_(index,index);
256  }
257  else if( IsLower_v<MT> )
258  {
259  const size_t begin( IsStrictlyLower_v<MT> ? index+1UL : index );
260  const size_t n ( mat_.rows() - begin );
261  return subvector( vec_, begin, n, unchecked ) *
262  subvector( column( mat_, index, unchecked ), begin, n, unchecked );
263  }
264  else if( IsUpper_v<MT> )
265  {
266  const size_t n( IsStrictlyUpper_v<MT> ? index : index+1UL );
267  return subvector( vec_, 0UL, n, unchecked ) *
268  subvector( column( mat_, index, unchecked ), 0UL, n, unchecked );
269  }
270  else
271  {
272  return vec_ * column( mat_, index, unchecked );
273  }
274  }
275  //**********************************************************************************************
276 
277  //**At function*********************************************************************************
284  inline ReturnType at( size_t index ) const {
285  if( index >= mat_.columns() ) {
286  BLAZE_THROW_OUT_OF_RANGE( "Invalid vector access index" );
287  }
288  return (*this)[index];
289  }
290  //**********************************************************************************************
291 
292  //**Size function*******************************************************************************
297  inline size_t size() const noexcept {
298  return mat_.columns();
299  }
300  //**********************************************************************************************
301 
302  //**Left operand access*************************************************************************
307  inline LeftOperand leftOperand() const noexcept {
308  return vec_;
309  }
310  //**********************************************************************************************
311 
312  //**Right operand access************************************************************************
317  inline RightOperand rightOperand() const noexcept {
318  return mat_;
319  }
320  //**********************************************************************************************
321 
322  //**********************************************************************************************
328  template< typename T >
329  inline bool canAlias( const T* alias ) const noexcept {
330  return vec_.isAliased( alias ) || mat_.isAliased( alias );
331  }
332  //**********************************************************************************************
333 
334  //**********************************************************************************************
340  template< typename T >
341  inline bool isAliased( const T* alias ) const noexcept {
342  return vec_.isAliased( alias ) || mat_.isAliased( alias );
343  }
344  //**********************************************************************************************
345 
346  //**********************************************************************************************
351  inline bool isAligned() const noexcept {
352  return mat_.isAligned();
353  }
354  //**********************************************************************************************
355 
356  //**********************************************************************************************
361  inline bool canSMPAssign() const noexcept {
362  return ( size() > SMP_TSVECDMATMULT_THRESHOLD );
363  }
364  //**********************************************************************************************
365 
366  private:
367  //**Member variables****************************************************************************
370  //**********************************************************************************************
371 
372  //**Assignment to dense vectors*****************************************************************
385  template< typename VT2 > // Type of the target dense vector
386  friend inline void assign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
387  {
389 
390  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
391 
392  // Evaluation of the left-hand side sparse vector operand
393  LT x( serial( rhs.vec_ ) );
394  if( x.nonZeros() == 0UL ) {
395  reset( ~lhs );
396  return;
397  }
398 
399  // Evaluation of the right-hand side dense matrix operand
400  RT A( serial( rhs.mat_ ) );
401 
402  // Checking the evaluated operands
403  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
404  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
405  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
406  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
407 
408  // Performing the sparse vector-dense matrix multiplication
409  TSVecDMatMultExpr::selectAssignKernel( ~lhs, x, A );
410  }
412  //**********************************************************************************************
413 
414  //**Default assignment to dense vectors*********************************************************
428  template< typename VT1 // Type of the left-hand side target vector
429  , typename VT2 // Type of the left-hand side vector operand
430  , typename MT1 > // Type of the right-hand side matrix operand
431  static inline auto selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
433  {
434  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
435 
436  const size_t N( A.columns() );
437 
438  auto element( x.begin() );
439  const auto end( x.end() );
440 
441  size_t last( 0UL );
442 
443  if( IsUpper_v<MT1> ) {
444  const size_t jend( IsStrictlyUpper_v<MT1> ? element->index()+1UL : element->index() );
445  for( size_t j=0UL; j<jend; ++j )
446  reset( y[j] );
447  }
448 
449  for( ; element!=end; ++element )
450  {
451  const size_t index( element->index() );
452 
453  if( IsDiagonal_v<MT1> )
454  {
455  for( size_t j=last; j<index; ++j )
456  reset( y[j] );
457 
458  y[index] = element->value() * A(index,index);
459  last = index + 1UL;
460  }
461  else
462  {
463  const size_t jbegin( ( IsUpper_v<MT1> )
464  ?( IsStrictlyUpper_v<MT1> ? index+1UL : index )
465  :( 0UL ) );
466  const size_t jend( ( IsLower_v<MT1> )
467  ?( IsStrictlyLower_v<MT1> ? index : index+1UL )
468  :( N ) );
469  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
470 
471  for( size_t j=jbegin; j<last; ++j ) {
472  y[j] += element->value() * A(index,j);
473  }
474  for( size_t j=last; j<jend; ++j ) {
475  y[j] = element->value() * A(index,j);
476  }
477 
478  last = jend;
479  }
480  }
481 
482  if( IsLower_v<MT1> ) {
483  for( size_t j=last; j<N; ++j )
484  reset( y[j] );
485  }
486  }
488  //**********************************************************************************************
489 
490  //**Optimized assignment to dense vectors*******************************************************
504  template< typename VT1 // Type of the left-hand side target vector
505  , typename VT2 // Type of the left-hand side vector operand
506  , typename MT1 > // Type of the right-hand side matrix operand
507  static inline auto selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
508  -> EnableIf_t< UseOptimizedKernel_v<VT1,VT2,MT1> >
509  {
510  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
511 
512  const size_t N( A.columns() );
513 
514  auto element( x.begin() );
515  const auto end( x.end() );
516 
517  const size_t ipos( x.nonZeros() & size_t(-4) );
518  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
519 
520  if( ipos > 3UL )
521  {
522  const size_t i1( element->index() );
523  const VET v1( element->value() );
524  ++element;
525  const size_t i2( element->index() );
526  const VET v2( element->value() );
527  ++element;
528  const size_t i3( element->index() );
529  const VET v3( element->value() );
530  ++element;
531  const size_t i4( element->index() );
532  const VET v4( element->value() );
533  ++element;
534 
535  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
536 
537  for( size_t j=0UL; j<N; ++j ) {
538  y[j] = v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
539  }
540  }
541  else
542  {
543  const size_t i1( element->index() );
544  const VET v1( element->value() );
545  ++element;
546 
547  for( size_t j=0UL; j<N; ++j ) {
548  y[j] = v1 * A(i1,j);
549  }
550  }
551 
552  for( size_t i=(ipos>3UL)?(4UL):(1UL); (i+4UL)<=ipos; i+=4UL )
553  {
554  const size_t i1( element->index() );
555  const VET v1( element->value() );
556  ++element;
557  const size_t i2( element->index() );
558  const VET v2( element->value() );
559  ++element;
560  const size_t i3( element->index() );
561  const VET v3( element->value() );
562  ++element;
563  const size_t i4( element->index() );
564  const VET v4( element->value() );
565  ++element;
566 
567  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
568 
569  const size_t jbegin( ( IsUpper_v<MT1> )
570  ?( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 )
571  :( 0UL ) );
572  const size_t jend( ( IsLower_v<MT1> )
573  ?( IsStrictlyLower_v<MT1> ? i4 : i4+1UL )
574  :( N ) );
575  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
576 
577  for( size_t j=jbegin; j<jend; ++j ) {
578  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
579  }
580  }
581  for( ; element!=end; ++element )
582  {
583  const size_t i1( element->index() );
584  const VET v1( element->value() );
585 
586  const size_t jbegin( ( IsUpper_v<MT1> )
587  ?( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 )
588  :( 0UL ) );
589  const size_t jend( ( IsLower_v<MT1> )
590  ?( IsStrictlyLower_v<MT1> ? i1 : i1+1UL )
591  :( N ) );
592  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
593 
594  for( size_t j=jbegin; j<jend; ++j ) {
595  y[j] += v1 * A(i1,j);
596  }
597  }
598  }
600  //**********************************************************************************************
601 
602  //**Vectorized assignment to dense vectors******************************************************
616  template< typename VT1 // Type of the left-hand side target vector
617  , typename VT2 // Type of the left-hand side vector operand
618  , typename MT1 > // Type of the right-hand side matrix operand
619  static inline auto selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
620  -> EnableIf_t< UseVectorizedKernel_v<VT1,VT2,MT1> >
621  {
622  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
623 
624  constexpr bool remainder( !IsPadded_v<VT1> || !IsPadded_v<MT1> );
625 
626  const size_t N( A.columns() );
627 
628  auto element( x.begin() );
629  const auto end( x.end() );
630 
631  const size_t ipos( x.nonZeros() & size_t(-4) );
632  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
633 
634  if( ipos > 3UL )
635  {
636  const size_t i1( element->index() );
637  const VET v1( element->value() );
638  ++element;
639  const size_t i2( element->index() );
640  const VET v2( element->value() );
641  ++element;
642  const size_t i3( element->index() );
643  const VET v3( element->value() );
644  ++element;
645  const size_t i4( element->index() );
646  const VET v4( element->value() );
647  ++element;
648 
649  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
650 
651  const SIMDType xmm1( set( v1 ) );
652  const SIMDType xmm2( set( v2 ) );
653  const SIMDType xmm3( set( v3 ) );
654  const SIMDType xmm4( set( v4 ) );
655 
656  const size_t jpos( remainder ? ( N & size_t(-SIMDSIZE) ) : N );
657  BLAZE_INTERNAL_ASSERT( !remainder || ( N - ( N % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
658 
659  size_t j( 0UL );
660 
661  for( ; j<jpos; j+=SIMDSIZE ) {
662  y.store( j, xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
663  }
664  for( ; remainder && j<N; ++j ) {
665  y[j] = v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
666  }
667  }
668  else
669  {
670  const size_t i1( element->index() );
671  const VET v1( element->value() );
672  ++element;
673 
674  const SIMDType xmm1( set( v1 ) );
675 
676  const size_t jpos( remainder ? ( N & size_t(-SIMDSIZE) ) : N );
677  BLAZE_INTERNAL_ASSERT( !remainder || ( N - ( N % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
678 
679  size_t j( 0UL );
680 
681  for( ; j<jpos; j+=SIMDSIZE ) {
682  y.store( j, xmm1 * A.load(i1,j) );
683  }
684  for( ; remainder && j<N; ++j ) {
685  y[j] = v1 * A(i1,j);
686  }
687  }
688 
689  for( size_t i=(ipos>3UL)?(4UL):(1UL); (i+4UL)<=ipos; i+=4UL )
690  {
691  const size_t i1( element->index() );
692  const VET v1( element->value() );
693  ++element;
694  const size_t i2( element->index() );
695  const VET v2( element->value() );
696  ++element;
697  const size_t i3( element->index() );
698  const VET v3( element->value() );
699  ++element;
700  const size_t i4( element->index() );
701  const VET v4( element->value() );
702  ++element;
703 
704  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
705 
706  const SIMDType xmm1( set( v1 ) );
707  const SIMDType xmm2( set( v2 ) );
708  const SIMDType xmm3( set( v3 ) );
709  const SIMDType xmm4( set( v4 ) );
710 
711  const size_t jbegin( ( IsUpper_v<MT1> )
712  ?( ( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
713  :( 0UL ) );
714  const size_t jend( ( IsLower_v<MT1> )
715  ?( IsStrictlyLower_v<MT1> ? i4 : i4+1UL )
716  :( N ) );
717  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
718 
719  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
720  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
721 
722  size_t j( jbegin );
723 
724  for( ; j<jpos; j+=SIMDSIZE ) {
725  y.store( j, y.load(j) + xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
726  }
727  for( ; remainder && j<jend; ++j ) {
728  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
729  }
730  }
731  for( ; element!=end; ++element )
732  {
733  const size_t i1( element->index() );
734  const VET v1( element->value() );
735 
736  const SIMDType xmm1( set( v1 ) );
737 
738  const size_t jbegin( ( IsUpper_v<MT1> )
739  ?( ( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
740  :( 0UL ) );
741  const size_t jend( ( IsLower_v<MT1> )
742  ?( IsStrictlyLower_v<MT1> ? i1 : i1+1UL )
743  :( N ) );
744  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
745 
746  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
747  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
748 
749  size_t j( jbegin );
750 
751  for( ; j<jpos; j+=SIMDSIZE ) {
752  y.store( j, y.load(j) + xmm1 * A.load(i1,j) );
753  }
754  for( ; remainder && j<jend; ++j ) {
755  y[j] += v1 * A(i1,j);
756  }
757  }
758  }
760  //**********************************************************************************************
761 
762  //**Assignment to sparse vectors****************************************************************
775  template< typename VT2 > // Type of the target sparse vector
776  friend inline void assign( SparseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
777  {
779 
783 
784  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
785 
786  const ResultType tmp( serial( rhs ) );
787  assign( ~lhs, tmp );
788  }
790  //**********************************************************************************************
791 
792  //**Addition assignment to dense vectors********************************************************
804  template< typename VT2 > // Type of the target dense vector
805  friend inline void addAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
806  {
808 
809  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
810 
811  // Evaluation of the left-hand side sparse vector operand
812  LT x( serial( rhs.vec_ ) );
813  if( x.nonZeros() == 0UL ) return;
814 
815  // Evaluation of the right-hand side dense matrix operand
816  RT A( serial( rhs.mat_ ) );
817 
818  // Checking the evaluated operands
819  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
820  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
821  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
822  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
823 
824  // Performing the sparse vector-dense matrix multiplication
825  TSVecDMatMultExpr::selectAddAssignKernel( ~lhs, x, A );
826  }
827  //**********************************************************************************************
828 
829  //**Default addition assignment to dense vectors************************************************
843  template< typename VT1 // Type of the left-hand side target vector
844  , typename VT2 // Type of the left-hand side vector operand
845  , typename MT1 > // Type of the right-hand side matrix operand
846  static inline auto selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
848  {
849  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
850 
851  const size_t N( A.columns() );
852 
853  auto element( x.begin() );
854  const auto end( x.end() );
855 
856  for( ; element!=end; ++element )
857  {
858  const size_t index( element->index() );
859 
860  if( IsDiagonal_v<MT1> )
861  {
862  y[index] += A(index,index) * element->value();
863  }
864  else
865  {
866  const size_t jbegin( ( IsUpper_v<MT1> )
867  ?( IsStrictlyUpper_v<MT1> ? index+1UL : index )
868  :( 0UL ) );
869  const size_t jend( ( IsLower_v<MT1> )
870  ?( IsStrictlyLower_v<MT1> ? index : index+1UL )
871  :( N ) );
872  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
873 
874  for( size_t j=jbegin; j<jend; ++j ) {
875  y[j] += element->value() * A(index,j);
876  }
877  }
878  }
879  }
881  //**********************************************************************************************
882 
883  //**Optimized addition assignment to dense vectors**********************************************
897  template< typename VT1 // Type of the left-hand side target vector
898  , typename VT2 // Type of the left-hand side vector operand
899  , typename MT1 > // Type of the right-hand side matrix operand
900  static inline auto selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
901  -> EnableIf_t< UseOptimizedKernel_v<VT1,VT2,MT1> >
902  {
903  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
904 
905  const size_t N( A.columns() );
906 
907  auto element( x.begin() );
908  const auto end( x.end() );
909 
910  const size_t ipos( x.nonZeros() & size_t(-4) );
911  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
912 
913  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
914  {
915  const size_t i1( element->index() );
916  const VET v1( element->value() );
917  ++element;
918  const size_t i2( element->index() );
919  const VET v2( element->value() );
920  ++element;
921  const size_t i3( element->index() );
922  const VET v3( element->value() );
923  ++element;
924  const size_t i4( element->index() );
925  const VET v4( element->value() );
926  ++element;
927 
928  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
929 
930  const size_t jbegin( ( IsUpper_v<MT1> )
931  ?( IsStrictlyUpper_v<MT1> ? i+1UL : i1 )
932  :( 0UL ) );
933  const size_t jend( ( IsLower_v<MT1> )
934  ?( IsStrictlyLower_v<MT1> ? i4 : i4+1UL )
935  :( N ) );
936  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
937 
938  for( size_t j=jbegin; j<jend; ++j ) {
939  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
940  }
941  }
942  for( ; element!=end; ++element )
943  {
944  const size_t i1( element->index() );
945  const VET v1( element->value() );
946 
947  const size_t jbegin( ( IsUpper_v<MT1> )
948  ?( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 )
949  :( 0UL ) );
950  const size_t jend( ( IsLower_v<MT1> )
951  ?( IsStrictlyLower_v<MT1> ? i1 : i1+1UL )
952  :( N ) );
953  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
954 
955  for( size_t j=jbegin; j<jend; ++j ) {
956  y[j] += v1 * A(i1,j);
957  }
958  }
959  }
961  //**********************************************************************************************
962 
963  //**Vectorized addition assignment to dense vectors*********************************************
977  template< typename VT1 // Type of the left-hand side target vector
978  , typename VT2 // Type of the left-hand side vector operand
979  , typename MT1 > // Type of the right-hand side matrix operand
980  static inline auto selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
981  -> EnableIf_t< UseVectorizedKernel_v<VT1,VT2,MT1> >
982  {
983  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
984 
985  constexpr bool remainder( !IsPadded_v<VT1> || !IsPadded_v<MT1> );
986 
987  const size_t N( A.columns() );
988 
989  auto element( x.begin() );
990  const auto end( x.end() );
991 
992  const size_t ipos( x.nonZeros() & size_t(-4) );
993  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
994 
995  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
996  {
997  const size_t i1( element->index() );
998  const VET v1( element->value() );
999  ++element;
1000  const size_t i2( element->index() );
1001  const VET v2( element->value() );
1002  ++element;
1003  const size_t i3( element->index() );
1004  const VET v3( element->value() );
1005  ++element;
1006  const size_t i4( element->index() );
1007  const VET v4( element->value() );
1008  ++element;
1009 
1010  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1011 
1012  const SIMDType xmm1( set( v1 ) );
1013  const SIMDType xmm2( set( v2 ) );
1014  const SIMDType xmm3( set( v3 ) );
1015  const SIMDType xmm4( set( v4 ) );
1016 
1017  const size_t jbegin( ( IsUpper_v<MT1> )
1018  ?( ( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1019  :( 0UL ) );
1020  const size_t jend( ( IsLower_v<MT1> )
1021  ?( IsStrictlyLower_v<MT1> ? i4 : i4+1UL )
1022  :( N ) );
1023  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1024 
1025  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1026  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1027 
1028  size_t j( jbegin );
1029 
1030  for( ; j<jpos; j+=SIMDSIZE ) {
1031  y.store( j, y.load(j) + xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
1032  }
1033  for( ; remainder && j<jend; ++j ) {
1034  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1035  }
1036  }
1037  for( ; element!=end; ++element )
1038  {
1039  const size_t i1( element->index() );
1040  const VET v1( element->value() );
1041 
1042  const SIMDType xmm1( set( v1 ) );
1043 
1044  const size_t jbegin( ( IsUpper_v<MT1> )
1045  ?( ( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1046  :( 0UL ) );
1047  const size_t jend( ( IsLower_v<MT1> )
1048  ?( IsStrictlyLower_v<MT1> ? i1 : i1+1UL )
1049  :( N ) );
1050  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1051 
1052  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1053  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1054 
1055  size_t j( jbegin );
1056 
1057  for( ; j<jpos; j+=SIMDSIZE ) {
1058  y.store( j, y.load(j) + xmm1 * A.load(i1,j) );
1059  }
1060  for( ; remainder && j<jend; ++j ) {
1061  y[j] += v1 * A(i1,j);
1062  }
1063  }
1064  }
1066  //**********************************************************************************************
1067 
1068  //**Addition assignment to sparse vectors*******************************************************
1069  // No special implementation for the addition assignment to sparse vectors.
1070  //**********************************************************************************************
1071 
1072  //**Subtraction assignment to dense vectors*****************************************************
1084  template< typename VT2 > // Type of the target dense vector
1085  friend inline void subAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1086  {
1088 
1089  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1090 
1091  // Evaluation of the left-hand side sparse vector operand
1092  LT x( serial( rhs.vec_ ) );
1093  if( x.nonZeros() == 0UL ) return;
1094 
1095  // Evaluation of the right-hand side dense matrix operand
1096  RT A( serial( rhs.mat_ ) );
1097 
1098  // Checking the evaluated operands
1099  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1100  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1101  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1102  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1103 
1104  // Performing the sparse vector-dense matrix multiplication
1105  TSVecDMatMultExpr::selectSubAssignKernel( ~lhs, x, A );
1106  }
1107  //**********************************************************************************************
1108 
1109  //**Default subtraction assignment to dense vectors*********************************************
1123  template< typename VT1 // Type of the left-hand side target vector
1124  , typename VT2 // Type of the left-hand side vector operand
1125  , typename MT1 > // Type of the right-hand side matrix operand
1126  static inline auto selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1128  {
1129  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1130 
1131  const size_t N( A.columns() );
1132 
1133  auto element( x.begin() );
1134  const auto end( x.end() );
1135 
1136  for( ; element!=end; ++element )
1137  {
1138  const size_t index( element->index() );
1139 
1140  if( IsDiagonal_v<MT1> )
1141  {
1142  y[index] -= A(index,index) * element->value();
1143  }
1144  else
1145  {
1146  const size_t jbegin( ( IsUpper_v<MT1> )
1147  ?( IsStrictlyUpper_v<MT1> ? index+1UL : index )
1148  :( 0UL ) );
1149  const size_t jend( ( IsLower_v<MT1> )
1150  ?( IsStrictlyLower_v<MT1> ? index : index+1UL )
1151  :( N ) );
1152  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1153 
1154  for( size_t j=jbegin; j<jend; ++j ) {
1155  y[j] -= element->value() * A(index,j);
1156  }
1157  }
1158  }
1159  }
1161  //**********************************************************************************************
1162 
1163  //**Optimized subtraction assignment to dense vectors*******************************************
1177  template< typename VT1 // Type of the left-hand side target vector
1178  , typename VT2 // Type of the left-hand side vector operand
1179  , typename MT1 > // Type of the right-hand side matrix operand
1180  static inline auto selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1181  -> EnableIf_t< UseOptimizedKernel_v<VT1,VT2,MT1> >
1182  {
1183  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1184 
1185  const size_t N( A.columns() );
1186 
1187  auto element( x.begin() );
1188  const auto end( x.end() );
1189 
1190  const size_t ipos( x.nonZeros() & size_t(-4) );
1191  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1192 
1193  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1194  {
1195  const size_t i1( element->index() );
1196  const VET v1( element->value() );
1197  ++element;
1198  const size_t i2( element->index() );
1199  const VET v2( element->value() );
1200  ++element;
1201  const size_t i3( element->index() );
1202  const VET v3( element->value() );
1203  ++element;
1204  const size_t i4( element->index() );
1205  const VET v4( element->value() );
1206  ++element;
1207 
1208  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1209 
1210  const size_t jbegin( ( IsUpper_v<MT1> )
1211  ?( IsStrictlyUpper_v<MT1> ? i+1UL : i1 )
1212  :( 0UL ) );
1213  const size_t jend( ( IsLower_v<MT1> )
1214  ?( IsStrictlyLower_v<MT1> ? i4 : i4+1UL )
1215  :( N ) );
1216  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1217 
1218  for( size_t j=jbegin; j<jend; ++j ) {
1219  y[j] -= v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1220  }
1221  }
1222  for( ; element!=end; ++element )
1223  {
1224  const size_t i1( element->index() );
1225  const VET v1( element->value() );
1226 
1227  const size_t jbegin( ( IsUpper_v<MT1> )
1228  ?( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 )
1229  :( 0UL ) );
1230  const size_t jend( ( IsLower_v<MT1> )
1231  ?( IsStrictlyLower_v<MT1> ? i1 : i1+1UL )
1232  :( N ) );
1233  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1234 
1235  for( size_t j=jbegin; j<jend; ++j ) {
1236  y[j] -= v1 * A(i1,j);
1237  }
1238  }
1239  }
1241  //**********************************************************************************************
1242 
1243  //**Vectorized subtraction assignment to dense vectors******************************************
1257  template< typename VT1 // Type of the left-hand side target vector
1258  , typename VT2 // Type of the left-hand side vector operand
1259  , typename MT1 > // Type of the right-hand side matrix operand
1260  static inline auto selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1261  -> EnableIf_t< UseVectorizedKernel_v<VT1,VT2,MT1> >
1262  {
1263  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1264 
1265  constexpr bool remainder( !IsPadded_v<VT1> || !IsPadded_v<MT1> );
1266 
1267  const size_t N( A.columns() );
1268 
1269  auto element( x.begin() );
1270  const auto end( x.end() );
1271 
1272  const size_t ipos( x.nonZeros() & size_t(-4) );
1273  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1274 
1275  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1276  {
1277  const size_t i1( element->index() );
1278  const VET v1( element->value() );
1279  ++element;
1280  const size_t i2( element->index() );
1281  const VET v2( element->value() );
1282  ++element;
1283  const size_t i3( element->index() );
1284  const VET v3( element->value() );
1285  ++element;
1286  const size_t i4( element->index() );
1287  const VET v4( element->value() );
1288  ++element;
1289 
1290  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1291 
1292  const SIMDType xmm1( set( v1 ) );
1293  const SIMDType xmm2( set( v2 ) );
1294  const SIMDType xmm3( set( v3 ) );
1295  const SIMDType xmm4( set( v4 ) );
1296 
1297  const size_t jbegin( ( IsUpper_v<MT1> )
1298  ?( ( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1299  :( 0UL ) );
1300  const size_t jend( ( IsLower_v<MT1> )
1301  ?( IsStrictlyLower_v<MT1> ? i4 : i4+1UL )
1302  :( N ) );
1303  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1304 
1305  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1306  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1307 
1308  size_t j( jbegin );
1309 
1310  for( ; j<jpos; j+=SIMDSIZE ) {
1311  y.store( j, y.load(j) - xmm1 * A.load(i1,j) - xmm2 * A.load(i2,j) - xmm3 * A.load(i3,j) - xmm4 * A.load(i4,j) );
1312  }
1313  for( ; remainder && j<jend; ++j ) {
1314  y[j] -= v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1315  }
1316  }
1317  for( ; element!=end; ++element )
1318  {
1319  const size_t i1( element->index() );
1320  const VET v1( element->value() );
1321 
1322  const SIMDType xmm1( set( v1 ) );
1323 
1324  const size_t jbegin( ( IsUpper_v<MT1> )
1325  ?( ( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1326  :( 0UL ) );
1327  const size_t jend( ( IsLower_v<MT1> )
1328  ?( IsStrictlyLower_v<MT1> ? i1 : i1+1UL )
1329  :( N ) );
1330  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1331 
1332  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1333  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1334 
1335  size_t j( jbegin );
1336 
1337  for( ; j<jpos; j+=SIMDSIZE ) {
1338  y.store( j, y.load(j) - xmm1 * A.load(i1,j) );
1339  }
1340  for( ; remainder && j<jend; ++j ) {
1341  y[j] -= v1 * A(i1,j);
1342  }
1343  }
1344  }
1346  //**********************************************************************************************
1347 
1348  //**Subtraction assignment to sparse vectors****************************************************
1349  // No special implementation for the subtraction assignment to sparse vectors.
1350  //**********************************************************************************************
1351 
1352  //**Multiplication assignment to dense vectors**************************************************
1364  template< typename VT2 > // Type of the target dense vector
1365  friend inline void multAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1366  {
1368 
1372 
1373  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1374 
1375  const ResultType tmp( serial( rhs ) );
1376  multAssign( ~lhs, tmp );
1377  }
1378  //**********************************************************************************************
1379 
1380  //**Multiplication assignment to sparse vectors*************************************************
1381  // No special implementation for the multiplication assignment to sparse vectors.
1382  //**********************************************************************************************
1383 
1384  //**Division assignment to dense vectors********************************************************
1396  template< typename VT2 > // Type of the target dense vector
1397  friend inline void divAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1398  {
1400 
1404 
1405  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1406 
1407  const ResultType tmp( serial( rhs ) );
1408  divAssign( ~lhs, tmp );
1409  }
1410  //**********************************************************************************************
1411 
1412  //**Division assignment to sparse vectors*******************************************************
1413  // No special implementation for the division assignment to sparse vectors.
1414  //**********************************************************************************************
1415 
1416  //**SMP assignment to dense vectors*************************************************************
1431  template< typename VT2 > // Type of the target dense vector
1432  friend inline auto smpAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1434  {
1436 
1437  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1438 
1439  // Evaluation of the left-hand side sparse vector operand
1440  LT x( rhs.vec_ );
1441  if( x.nonZeros() == 0UL ) {
1442  reset( ~lhs );
1443  return;
1444  }
1445 
1446  // Evaluation of the right-hand side dense matrix operand
1447  RT A( rhs.mat_ );
1448 
1449  // Checking the evaluated operands
1450  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1451  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1452  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1453  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1454 
1455  // Performing the sparse vector-dense matrix multiplication
1456  smpAssign( ~lhs, x * A );
1457  }
1459  //**********************************************************************************************
1460 
1461  //**SMP assignment to sparse vectors************************************************************
1476  template< typename VT2 > // Type of the target sparse vector
1477  friend inline auto smpAssign( SparseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1478  -> EnableIf_t< UseSMPAssign_v<VT2> >
1479  {
1481 
1485 
1486  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1487 
1488  const ResultType tmp( rhs );
1489  smpAssign( ~lhs, tmp );
1490  }
1492  //**********************************************************************************************
1493 
1494  //**SMP addition assignment to dense vectors****************************************************
1508  template< typename VT2 > // Type of the target dense vector
1509  friend inline auto smpAddAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1511  {
1513 
1514  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1515 
1516  // Evaluation of the left-hand side sparse vector operand
1517  LT x( rhs.vec_ );
1518  if( x.nonZeros() == 0UL ) return;
1519 
1520  // Evaluation of the right-hand side dense matrix operand
1521  RT A( rhs.mat_ );
1522 
1523  // Checking the evaluated operands
1524  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1525  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1526  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1527  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1528 
1529  // Performing the sparse vector-dense matrix multiplication
1530  smpAddAssign( ~lhs, x * A );
1531  }
1532  //**********************************************************************************************
1533 
1534  //**SMP addition assignment to sparse vectors***************************************************
1535  // No special implementation for the SMP addition assignment to sparse vectors.
1536  //**********************************************************************************************
1537 
1538  //**SMP subtraction assignment to dense vectors*************************************************
1552  template< typename VT2 > // Type of the target dense vector
1553  friend inline auto smpSubAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1555  {
1557 
1558  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1559 
1560  // Evaluation of the left-hand side sparse vector operand
1561  LT x( rhs.vec_ );
1562  if( x.nonZeros() == 0UL ) return;
1563 
1564  // Evaluation of the right-hand side dense matrix operand
1565  RT A( rhs.mat_ );
1566 
1567  // Checking the evaluated operands
1568  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1569  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1570  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1571  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1572 
1573  // Performing the sparse vector-dense matrix multiplication
1574  smpSubAssign( ~lhs, x * A );
1575  }
1576  //**********************************************************************************************
1577 
1578  //**SMP subtraction assignment to sparse vectors************************************************
1579  // No special implementation for the SMP subtraction assignment to sparse vectors.
1580  //**********************************************************************************************
1581 
1582  //**SMP multiplication assignment to dense vectors**********************************************
1596  template< typename VT2 > // Type of the target dense vector
1597  friend inline auto smpMultAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1599  {
1601 
1605 
1606  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1607 
1608  const ResultType tmp( rhs );
1609  smpMultAssign( ~lhs, tmp );
1610  }
1611  //**********************************************************************************************
1612 
1613  //**SMP multiplication assignment to sparse vectors*********************************************
1614  // No special implementation for the SMP multiplication assignment to sparse vectors.
1615  //**********************************************************************************************
1616 
1617  //**SMP division assignment to dense vectors****************************************************
1631  template< typename VT2 > // Type of the target dense vector
1632  friend inline auto smpDivAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1634  {
1636 
1640 
1641  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1642 
1643  const ResultType tmp( rhs );
1644  smpDivAssign( ~lhs, tmp );
1645  }
1646  //**********************************************************************************************
1647 
1648  //**SMP division assignment to sparse vectors***************************************************
1649  // No special implementation for the SMP division assignment to sparse vectors.
1650  //**********************************************************************************************
1651 
1652  //**Compile time checks*************************************************************************
1661  //**********************************************************************************************
1662 };
1663 //*************************************************************************************************
1664 
1665 
1666 
1667 
1668 //=================================================================================================
1669 //
1670 // GLOBAL BINARY ARITHMETIC OPERATORS
1671 //
1672 //=================================================================================================
1673 
1674 //*************************************************************************************************
1687 template< typename VT // Type of the left-hand side sparse vector
1688  , typename MT // Type of the right-hand side dense matrix
1689  , DisableIf_t< IsZero_v<VT> >* = nullptr >
1690 inline const TSVecDMatMultExpr<VT,MT>
1691  tsvecdmatmult( const SparseVector<VT,true>& vec, const DenseMatrix<MT,false>& mat )
1692 {
1694 
1695  BLAZE_INTERNAL_ASSERT( (~vec).size() == (~mat).rows(), "Invalid vector and matrix sizes" );
1696 
1697  return TSVecDMatMultExpr<VT,MT>( ~vec, ~mat );
1698 }
1700 //*************************************************************************************************
1701 
1702 
1703 //*************************************************************************************************
1716 template< typename VT // Type of the left-hand side sparse vector
1717  , typename MT // Type of the right-hand side dense matrix
1718  , EnableIf_t< IsZero_v<VT> >* = nullptr >
1719 inline decltype(auto)
1720  tsvecdmatmult( const SparseVector<VT,true>& vec, const DenseMatrix<MT,false>& mat )
1721 {
1723 
1724  MAYBE_UNUSED( vec );
1725 
1726  BLAZE_INTERNAL_ASSERT( (~vec).size() == (~mat).rows(), "Invalid vector and matrix sizes" );
1727 
1728  using ReturnType = const MultTrait_t< ResultType_t<VT>, ResultType_t<MT> >;
1729 
1731  BLAZE_CONSTRAINT_MUST_BE_ZERO_TYPE( ReturnType );
1732 
1733  return ReturnType( (~mat).columns() );
1734 }
1736 //*************************************************************************************************
1737 
1738 
1739 //*************************************************************************************************
1770 template< typename VT // Type of the left-hand side sparse vector
1771  , typename MT > // Type of the right-hand side dense matrix
1772 inline decltype(auto)
1773  operator*( const SparseVector<VT,true>& vec, const DenseMatrix<MT,false>& mat )
1774 {
1776 
1778 
1779  if( (~vec).size() != (~mat).rows() ) {
1780  BLAZE_THROW_INVALID_ARGUMENT( "Vector and matrix sizes do not match" );
1781  }
1782 
1783  return tsvecdmatmult( ~vec, ~mat );
1784 }
1785 //*************************************************************************************************
1786 
1787 
1788 
1789 
1790 //=================================================================================================
1791 //
1792 // GLOBAL RESTRUCTURING BINARY ARITHMETIC OPERATORS
1793 //
1794 //=================================================================================================
1795 
1796 //*************************************************************************************************
1810 template< typename VT // Type of the left-hand side sparse vector
1811  , typename MT > // Matrix base type of the right-hand side expression
1812 inline decltype(auto)
1813  operator*( const SparseVector<VT,true>& vec, const MatMatMultExpr<MT>& mat )
1814 {
1816 
1817  return ( vec * (~mat).leftOperand() ) * (~mat).rightOperand();
1818 }
1820 //*************************************************************************************************
1821 
1822 
1823 
1824 
1825 //=================================================================================================
1826 //
1827 // ISALIGNED SPECIALIZATIONS
1828 //
1829 //=================================================================================================
1830 
1831 //*************************************************************************************************
1833 template< typename VT, typename MT >
1834 struct IsAligned< TSVecDMatMultExpr<VT,MT> >
1835  : public IsAligned<MT>
1836 {};
1838 //*************************************************************************************************
1839 
1840 } // namespace blaze
1841 
1842 #endif
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Header file for auxiliary alias declarations.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:133
Header file for the blaze::checked and blaze::unchecked instances.
size_t size() const noexcept
Returns the current size/dimension of the vector.
Definition: TSVecDMatMultExpr.h:297
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TSVecDMatMultExpr.h:341
friend void multAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Multiplication assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ...
Definition: TSVecDMatMultExpr.h:1365
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:196
Header file for basic type definitions.
friend auto smpSubAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs) -> EnableIf_t< UseSMPAssign_v< VT2 > >
SMP subtraction assignment of a transpose sparse vector-dense matrix multiplication to a dense vector...
Definition: TSVecDMatMultExpr.h:1553
Expression object for transpose sparse vector-dense matrix multiplications.The TSVecDMatMultExpr clas...
Definition: Forward.h:194
typename If< Condition, T1, T2 >::Type If_t
Auxiliary alias template for the If class template.The If_t alias template provides a convenient shor...
Definition: If.h:109
ElementType_t< MRT > MET
Element type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:116
typename T::ResultType ResultType_t
Alias declaration for nested ResultType type definitions.The ResultType_t alias declaration provides ...
Definition: Aliases.h:390
Header file for the serial shim.
SIMDTrait_t< ElementType > SIMDType
Resulting SIMD element type.
Definition: TSVecDMatMultExpr.h:195
Header file for the IsDiagonal type trait.
If_t< IsExpression_v< MT >, const MT, const MT & > RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TSVecDMatMultExpr.h:203
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type,...
Definition: DenseMatrix.h:61
friend auto smpAddAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs) -> EnableIf_t< UseSMPAssign_v< VT2 > >
SMP addition assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ...
Definition: TSVecDMatMultExpr.h:1509
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:372
LeftOperand vec_
Left-hand side sparse vector of the multiplication expression.
Definition: TSVecDMatMultExpr.h:368
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:595
constexpr Unchecked unchecked
Global Unchecked instance.The blaze::unchecked instance is an optional token for the creation of view...
Definition: Check.h:138
Constraint on the data type.
typename SIMDTrait< T >::Type SIMDTrait_t
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_t alias declaration provid...
Definition: SIMDTrait.h:315
Header file for the DenseVector base class.
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:154
Header file for the MAYBE_UNUSED function template.
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TSVecDMatMultExpr.h:329
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Header file for the reset shim.
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
static constexpr bool evaluateMatrix
Compilation switch for the composite type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:128
constexpr size_t columns(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of columns of the matrix.
Definition: Matrix.h:514
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes....
Definition: DenseMatrix.h:81
static constexpr bool evaluateVector
Compilation switch for the composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:123
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.The ElementType_t alias declaration provide...
Definition: Aliases.h:170
If_t< evaluateVector, const VRT, VCT > LT
Type for the assignment of the left-hand side sparse vector operand.
Definition: TSVecDMatMultExpr.h:206
Constraint on the data type.
typename EnableIf< Condition, T >::Type EnableIf_t
Auxiliary type for the EnableIf class template.The EnableIf_t alias declaration provides a convenient...
Definition: EnableIf.h:138
RightOperand mat_
Right-hand side dense matrix of the multiplication expression.
Definition: TSVecDMatMultExpr.h:369
Header file for the DisableIf class template.
friend void addAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Addition assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ).
Definition: TSVecDMatMultExpr.h:805
Header file for the multiplication trait.
RightOperand rightOperand() const noexcept
Returns the right-hand side dense matrix operand.
Definition: TSVecDMatMultExpr.h:317
Header file for the IsStrictlyUpper type trait.
LeftOperand leftOperand() const noexcept
Returns the left-hand side sparse vector operand.
Definition: TSVecDMatMultExpr.h:307
Namespace of the Blaze C++ math library.
Definition: Blaze.h:58
Header file for the If class template.
friend void divAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Division assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ).
Definition: TSVecDMatMultExpr.h:1397
TransposeType_t< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:193
#define BLAZE_CONSTRAINT_MUST_BE_ZERO_TYPE(T)
Constraint on the data type.In case the given data type T is not a zero vector or matrix type,...
Definition: Zero.h:61
const ResultType CompositeType
Data type for composite expression templates.
Definition: TSVecDMatMultExpr.h:197
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
friend auto smpDivAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs) -> EnableIf_t< UseSMPAssign_v< VT2 > >
SMP division assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ...
Definition: TSVecDMatMultExpr.h:1632
Header file for all SIMD functionality.
Base class for N-dimensional dense vectors.The DenseVector class is a base class for all arbitrarily ...
Definition: DenseVector.h:76
If_t< evaluateMatrix, const MRT, MCT > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: TSVecDMatMultExpr.h:209
Header file for the IsLower type trait.
constexpr void MAYBE_UNUSED(const Args &...)
Suppression of unused parameter warnings.
Definition: MaybeUnused.h:81
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional vector type,...
Definition: SparseVector.h:61
Header file for the IsAligned type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_MATMATMULTEXPR_TYPE(T)
Constraint on the data type.In case the given data type T is a matrix/matrix multiplication expressio...
Definition: MatMatMultExpr.h:83
static constexpr size_t SIMDSIZE
The number of elements packed within a single SIMD element.
Definition: TSVecDMatMultExpr.h:227
Constraint on the data type.
Header file for the exception macros of the math module.
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:438
Constraint on the data type.
Constraint on the data type.
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
Header file for the IsPadded type trait.
ElementType_t< VRT > VET
Element type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:115
ResultType_t< MT > MRT
Result type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:114
typename MultTrait< T1, T2 >::Type MultTrait_t
Auxiliary alias declaration for the MultTrait class template.The MultTrait_t alias declaration provid...
Definition: MultTrait.h:240
ElementType_t< ResultType > ElementType
Resulting element type.
Definition: TSVecDMatMultExpr.h:194
Header file for the IsSIMDCombinable type trait.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TSVecDMatMultExpr.h:351
typename T::TransposeType TransposeType_t
Alias declaration for nested TransposeType type definitions.The TransposeType_t alias declaration pro...
Definition: Aliases.h:470
Header file for run time assertion macros.
typename T::CompositeType CompositeType_t
Alias declaration for nested CompositeType type definitions.The CompositeType_t alias declaration pro...
Definition: Aliases.h:90
If_t< IsExpression_v< VT >, const VT, const VT & > LeftOperand
Composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:200
Header file for the IsZero type trait.
Constraint on the data type.
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:295
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:94
Header file for all forward declarations for expression class templates.
ReturnType operator[](size_t index) const
Subscript operator for the direct access to the vector elements.
Definition: TSVecDMatMultExpr.h:250
BLAZE_ALWAYS_INLINE const EnableIf_t< IsIntegral_v< T > &&HasSize_v< T, 1UL >, If_t< IsSigned_v< T >, SIMDint8, SIMDuint8 > > set(T value) noexcept
Sets all values in the vector to the given 1-byte integral value.
Definition: Set.h:75
constexpr size_t size(const Matrix< MT, SO > &matrix) noexcept
Returns the total number of elements of the matrix.
Definition: Matrix.h:530
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
Header file for the TVecMatMultExpr base class.
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:808
TSVecDMatMultExpr(const VT &vec, const MT &mat) noexcept
Constructor for the TSVecDMatMultExpr class.
Definition: TSVecDMatMultExpr.h:236
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TSVecDMatMultExpr.h:361
constexpr size_t rows(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of rows of the matrix.
Definition: Matrix.h:498
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional vector type,...
Definition: DenseVector.h:61
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_TVECMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid vector/matrix ...
Definition: TVecMatMultExpr.h:104
CompositeType_t< MT > MCT
Composite type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:118
Header file for the IsComputation type trait class.
static constexpr bool smpAssignable
Compilation switch for the expression template assignment strategy.
Definition: TSVecDMatMultExpr.h:221
Base class for sparse vectors.The SparseVector class is a base class for all arbitrarily sized (N-dim...
Definition: Forward.h:146
ResultType_t< VT > VRT
Result type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:113
#define BLAZE_CONSTRAINT_MUST_BE_ROW_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a row dense or sparse vector type (i...
Definition: RowVector.h:61
static constexpr bool simdEnabled
Compilation switch for the expression template evaluation strategy.
Definition: TSVecDMatMultExpr.h:214
friend void subAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Subtraction assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:1085
Header file for the IsUpper type trait.
Constraint on the data type.
ReturnType at(size_t index) const
Checked access to the vector elements.
Definition: TSVecDMatMultExpr.h:284
MultTrait_t< VRT, MRT > ResultType
Result type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:192
CompositeType_t< VT > VCT
Composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:117
Header file for the IsResizable type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_ZERO_TYPE(T)
Constraint on the data type.In case the given data type T is a zero vector or matrix type,...
Definition: Zero.h:81
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression,...
Definition: Assert.h:101
friend auto smpMultAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs) -> EnableIf_t< UseSMPAssign_v< VT2 > >
SMP multiplication assignment of a transpose sparse vector-dense matrix multiplication to a dense vec...
Definition: TSVecDMatMultExpr.h:1597
Constraint on the transpose flag of vector types.
Header file for the IsExpression type trait class.
Header file for the function trace functionality.