TDMatSVecMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATSVECMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATSVECMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
54 #include <blaze/math/Exception.h>
59 #include <blaze/math/shims/Reset.h>
61 #include <blaze/math/SIMD.h>
78 #include <blaze/math/views/Check.h>
81 #include <blaze/util/Assert.h>
82 #include <blaze/util/DisableIf.h>
83 #include <blaze/util/EnableIf.h>
85 #include <blaze/util/mpl/If.h>
86 #include <blaze/util/Types.h>
87 #include <blaze/util/Unused.h>
88 
89 
90 namespace blaze {
91 
92 //=================================================================================================
93 //
94 // CLASS TDMATSVECMULTEXPR
95 //
96 //=================================================================================================
97 
98 //*************************************************************************************************
105 template< typename MT // Type of the left-hand side dense matrix
106  , typename VT > // Type of the right-hand side sparse vector
107 class TDMatSVecMultExpr
108  : public MatVecMultExpr< DenseVector< TDMatSVecMultExpr<MT,VT>, false > >
109  , private Computation
110 {
111  private:
112  //**Type definitions****************************************************************************
119  //**********************************************************************************************
120 
121  //**********************************************************************************************
123  static constexpr bool evaluateMatrix = RequiresEvaluation_v<MT>;
124  //**********************************************************************************************
125 
126  //**********************************************************************************************
128  static constexpr bool evaluateVector = ( IsComputation_v<VT> || RequiresEvaluation_v<VT> );
129  //**********************************************************************************************
130 
131  //**********************************************************************************************
133 
137  template< typename T1 >
138  static constexpr bool UseSMPAssign_v = ( evaluateMatrix || evaluateVector );
140  //**********************************************************************************************
141 
142  //**********************************************************************************************
144 
148  template< typename T1, typename T2, typename T3 >
149  static constexpr bool UseVectorizedKernel_v =
150  ( useOptimizedKernels &&
151  !IsDiagonal_v<T2> &&
152  T1::simdEnabled && T2::simdEnabled &&
153  IsSIMDCombinable_v< ElementType_t<T1>
155  , ElementType_t<T3> > &&
156  HasSIMDAdd_v< ElementType_t<T2>, ElementType_t<T3> > &&
157  HasSIMDMult_v< ElementType_t<T2>, ElementType_t<T3> > );
159  //**********************************************************************************************
160 
161  //**********************************************************************************************
163 
167  template< typename T1, typename T2, typename T3 >
168  static constexpr bool UseOptimizedKernel_v =
169  ( !UseVectorizedKernel_v<T1,T2,T3> &&
170  !IsDiagonal_v<T2> &&
171  !IsResizable_v< ElementType_t<T1> > &&
172  !IsResizable_v<VET> );
174  //**********************************************************************************************
175 
176  //**********************************************************************************************
178 
181  template< typename T1, typename T2, typename T3 >
182  static constexpr bool UseDefaultKernel_v =
183  ( !UseVectorizedKernel_v<T1,T2,T3> && !UseOptimizedKernel_v<T1,T2,T3> );
185  //**********************************************************************************************
186 
187  public:
188  //**Type definitions****************************************************************************
195  using ReturnType = const ElementType;
196  using CompositeType = const ResultType;
197 
199  using LeftOperand = If_t< IsExpression_v<MT>, const MT, const MT& >;
200 
202  using RightOperand = If_t< IsExpression_v<VT>, const VT, const VT& >;
203 
206 
209  //**********************************************************************************************
210 
211  //**Compilation flags***************************************************************************
213  static constexpr bool simdEnabled =
214  ( !IsDiagonal_v<MT> &&
215  MT::simdEnabled &&
216  HasSIMDAdd_v<MET,VET> &&
217  HasSIMDMult_v<MET,VET> );
218 
220  static constexpr bool smpAssignable =
222  //**********************************************************************************************
223 
224  //**SIMD properties*****************************************************************************
226  static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
227  //**********************************************************************************************
228 
229  //**Constructor*********************************************************************************
235  explicit inline TDMatSVecMultExpr( const MT& mat, const VT& vec ) noexcept
236  : mat_( mat ) // Left-hand side dense matrix of the multiplication expression
237  , vec_( vec ) // Right-hand side sparse vector of the multiplication expression
238  {
239  BLAZE_INTERNAL_ASSERT( mat_.columns() == vec_.size(), "Invalid matrix and vector sizes" );
240  }
241  //**********************************************************************************************
242 
243  //**Subscript operator**************************************************************************
249  inline ReturnType operator[]( size_t index ) const {
250  BLAZE_INTERNAL_ASSERT( index < mat_.rows(), "Invalid vector access index" );
251 
252  if( IsDiagonal_v<MT> )
253  {
254  return mat_(index,index) * vec_[index];
255  }
256  else if( IsLower_v<MT> )
257  {
258  const size_t n( IsStrictlyLower_v<MT> ? index : index+1UL );
259  return subvector( row( mat_, index, unchecked ), 0UL, n, unchecked ) *
260  subvector( vec_, 0UL, n, unchecked );
261  }
262  else if( IsUpper_v<MT> )
263  {
264  const size_t begin( IsStrictlyUpper_v<MT> ? index+1UL : index );
265  const size_t n ( mat_.columns() - begin );
266  return subvector( row( mat_, index, unchecked ), begin, n, unchecked ) *
267  subvector( vec_, begin, n, unchecked );
268  }
269  else
270  {
271  return row( mat_, index, unchecked ) * vec_;
272  }
273  }
274  //**********************************************************************************************
275 
276  //**At function*********************************************************************************
283  inline ReturnType at( size_t index ) const {
284  if( index >= mat_.rows() ) {
285  BLAZE_THROW_OUT_OF_RANGE( "Invalid vector access index" );
286  }
287  return (*this)[index];
288  }
289  //**********************************************************************************************
290 
291  //**Size function*******************************************************************************
296  inline size_t size() const noexcept {
297  return mat_.rows();
298  }
299  //**********************************************************************************************
300 
301  //**Left operand access*************************************************************************
306  inline LeftOperand leftOperand() const noexcept {
307  return mat_;
308  }
309  //**********************************************************************************************
310 
311  //**Right operand access************************************************************************
316  inline RightOperand rightOperand() const noexcept {
317  return vec_;
318  }
319  //**********************************************************************************************
320 
321  //**********************************************************************************************
327  template< typename T >
328  inline bool canAlias( const T* alias ) const noexcept {
329  return mat_.isAliased( alias ) || vec_.isAliased( alias );
330  }
331  //**********************************************************************************************
332 
333  //**********************************************************************************************
339  template< typename T >
340  inline bool isAliased( const T* alias ) const noexcept {
341  return mat_.isAliased( alias ) || vec_.isAliased( alias );
342  }
343  //**********************************************************************************************
344 
345  //**********************************************************************************************
350  inline bool isAligned() const noexcept {
351  return mat_.isAligned();
352  }
353  //**********************************************************************************************
354 
355  //**********************************************************************************************
360  inline bool canSMPAssign() const noexcept {
361  return ( size() > SMP_TDMATSVECMULT_THRESHOLD );
362  }
363  //**********************************************************************************************
364 
365  private:
366  //**Member variables****************************************************************************
369  //**********************************************************************************************
370 
371  //**Assignment to dense vectors*****************************************************************
384  template< typename VT1 > // Type of the target dense vector
385  friend inline void assign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
386  {
388 
389  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
390 
391  // Evaluation of the right-hand side sparse vector operand
392  RT x( serial( rhs.vec_ ) );
393  if( x.nonZeros() == 0UL ) {
394  reset( ~lhs );
395  return;
396  }
397 
398  // Evaluation of the left-hand side dense matrix operand
399  LT A( serial( rhs.mat_ ) );
400 
401  // Checking the evaluated operands
402  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
403  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
404  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
405  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
406 
407  // Performing the dense matrix-sparse vector multiplication
408  TDMatSVecMultExpr::selectAssignKernel( ~lhs, A, x );
409  }
411  //**********************************************************************************************
412 
413  //**Default assignment to dense vectors*********************************************************
427  template< typename VT1 // Type of the left-hand side target vector
428  , typename MT1 // Type of the left-hand side matrix operand
429  , typename VT2 > // Type of the right-hand side vector operand
430  static inline auto selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
432  {
433  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
434 
435  const size_t M( A.rows() );
436 
437  auto element( x.begin() );
438  const auto end( x.end() );
439 
440  size_t last( 0UL );
441 
442  if( IsLower_v<MT1> ) {
443  const size_t iend( IsStrictlyLower_v<MT1> ? element->index()+1UL : element->index() );
444  for( size_t i=0UL; i<iend; ++i )
445  reset( y[i] );
446  }
447 
448  for( ; element!=end; ++element )
449  {
450  const size_t index( element->index() );
451 
452  if( IsDiagonal_v<MT1> )
453  {
454  for( size_t i=last; i<index; ++i )
455  reset( y[i] );
456 
457  y[index] = A(index,index) * element->value();
458  last = index + 1UL;
459  }
460  else
461  {
462  const size_t ibegin( ( IsLower_v<MT1> )
463  ?( IsStrictlyLower_v<MT1> ? index+1UL : index )
464  :( 0UL ) );
465  const size_t iend( ( IsUpper_v<MT1> )
466  ?( IsStrictlyUpper_v<MT1> ? index : index+1UL )
467  :( M ) );
468  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
469 
470  for( size_t i=ibegin; i<last; ++i ) {
471  y[i] += A(i,index) * element->value();
472  }
473  for( size_t i=last; i<iend; ++i ) {
474  y[i] = A(i,index) * element->value();
475  }
476 
477  last = iend;
478  }
479  }
480 
481  if( IsUpper_v<MT1> ) {
482  for( size_t i=last; i<M; ++i )
483  reset( y[i] );
484  }
485  }
487  //**********************************************************************************************
488 
489  //**Optimized assignment to dense vectors*******************************************************
503  template< typename VT1 // Type of the left-hand side target vector
504  , typename MT1 // Type of the left-hand side matrix operand
505  , typename VT2 > // Type of the right-hand side vector operand
506  static inline auto selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
507  -> EnableIf_t< UseOptimizedKernel_v<VT1,MT1,VT2> >
508  {
509  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
510 
511  const size_t M( A.rows() );
512 
513  auto element( x.begin() );
514  const auto end( x.end() );
515 
516  const size_t jpos( x.nonZeros() & size_t(-4) );
517  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
518 
519  if( jpos > 3UL )
520  {
521  const size_t j1( element->index() );
522  const VET v1( element->value() );
523  ++element;
524  const size_t j2( element->index() );
525  const VET v2( element->value() );
526  ++element;
527  const size_t j3( element->index() );
528  const VET v3( element->value() );
529  ++element;
530  const size_t j4( element->index() );
531  const VET v4( element->value() );
532  ++element;
533 
534  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
535 
536  for( size_t i=0UL; i<M; ++i ) {
537  y[i] = A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
538  }
539  }
540  else
541  {
542  const size_t j1( element->index() );
543  const VET v1( element->value() );
544  ++element;
545 
546  for( size_t i=0UL; i<M; ++i ) {
547  y[i] = A(i,j1) * v1;
548  }
549  }
550 
551  for( size_t j=(jpos>3UL)?(4UL):(1UL); (j+4UL)<=jpos; j+=4UL )
552  {
553  const size_t j1( element->index() );
554  const VET v1( element->value() );
555  ++element;
556  const size_t j2( element->index() );
557  const VET v2( element->value() );
558  ++element;
559  const size_t j3( element->index() );
560  const VET v3( element->value() );
561  ++element;
562  const size_t j4( element->index() );
563  const VET v4( element->value() );
564  ++element;
565 
566  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
567 
568  const size_t ibegin( ( IsLower_v<MT1> )
569  ?( IsStrictlyLower_v<MT1> ? j1+1UL : j1 )
570  :( 0UL ) );
571  const size_t iend( ( IsUpper_v<MT1> )
572  ?( IsStrictlyUpper_v<MT1> ? j4 : j4+1UL )
573  :( M ) );
574  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
575 
576  for( size_t i=ibegin; i<iend; ++i ) {
577  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
578  }
579  }
580  for( ; element!=end; ++element )
581  {
582  const size_t j1( element->index() );
583  const VET v1( element->value() );
584 
585  const size_t ibegin( ( IsLower_v<MT1> )
586  ?( IsStrictlyLower_v<MT1> ? j1+1UL : j1 )
587  :( 0UL ) );
588  const size_t iend( ( IsUpper_v<MT1> )
589  ?( IsStrictlyUpper_v<MT1> ? j1 : j1+1UL )
590  :( M ) );
591  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
592 
593  for( size_t i=ibegin; i<iend; ++i ) {
594  y[i] += A(i,j1) * v1;
595  }
596  }
597  }
599  //**********************************************************************************************
600 
601  //**Vectorized assignment to dense vectors******************************************************
615  template< typename VT1 // Type of the left-hand side target vector
616  , typename MT1 // Type of the left-hand side matrix operand
617  , typename VT2 > // Type of the right-hand side vector operand
618  static inline auto selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
619  -> EnableIf_t< UseVectorizedKernel_v<VT1,MT1,VT2> >
620  {
621  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
622 
623  constexpr bool remainder( !IsPadded_v<MT1> || !IsPadded_v<VT1> );
624 
625  const size_t M( A.rows() );
626 
627  auto element( x.begin() );
628  const auto end( x.end() );
629 
630  const size_t jpos( x.nonZeros() & size_t(-4) );
631  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
632 
633  if( jpos > 3UL )
634  {
635  const size_t j1( element->index() );
636  const VET v1( element->value() );
637  ++element;
638  const size_t j2( element->index() );
639  const VET v2( element->value() );
640  ++element;
641  const size_t j3( element->index() );
642  const VET v3( element->value() );
643  ++element;
644  const size_t j4( element->index() );
645  const VET v4( element->value() );
646  ++element;
647 
648  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
649 
650  const SIMDType xmm1( set( v1 ) );
651  const SIMDType xmm2( set( v2 ) );
652  const SIMDType xmm3( set( v3 ) );
653  const SIMDType xmm4( set( v4 ) );
654 
655  const size_t ipos( remainder ? ( M & size_t(-SIMDSIZE) ) : M );
656  BLAZE_INTERNAL_ASSERT( !remainder || ( M - ( M % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
657 
658  size_t i( 0UL );
659 
660  for( ; i<ipos; i+=SIMDSIZE ) {
661  y.store( i, A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
662  }
663  for( ; remainder && i<M; ++i ) {
664  y[i] = A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
665  }
666  }
667  else
668  {
669  const size_t j1( element->index() );
670  const VET v1( element->value() );
671  ++element;
672 
673  const SIMDType xmm1( set( v1 ) );
674 
675  const size_t ipos( remainder ? ( M & size_t(-SIMDSIZE) ) : M );
676  BLAZE_INTERNAL_ASSERT( !remainder || ( M - ( M % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
677 
678  size_t i( 0UL );
679 
680  for( ; i<ipos; i+=SIMDSIZE ) {
681  y.store( i, A.load(i,j1) * xmm1 );
682  }
683  for( ; remainder && i<M; ++i ) {
684  y[i] = A(i,j1) * v1;
685  }
686  }
687 
688  for( size_t j=(jpos>3UL)?(4UL):(1UL); (j+4UL)<=jpos; j+=4UL )
689  {
690  const size_t j1( element->index() );
691  const VET v1( element->value() );
692  ++element;
693  const size_t j2( element->index() );
694  const VET v2( element->value() );
695  ++element;
696  const size_t j3( element->index() );
697  const VET v3( element->value() );
698  ++element;
699  const size_t j4( element->index() );
700  const VET v4( element->value() );
701  ++element;
702 
703  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
704 
705  const SIMDType xmm1( set( v1 ) );
706  const SIMDType xmm2( set( v2 ) );
707  const SIMDType xmm3( set( v3 ) );
708  const SIMDType xmm4( set( v4 ) );
709 
710  const size_t ibegin( ( IsLower_v<MT1> )
711  ?( ( IsStrictlyLower_v<MT1> ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
712  :( 0UL ) );
713  const size_t iend( ( IsUpper_v<MT1> )
714  ?( IsStrictlyUpper_v<MT1> ? j4 : j4+1UL )
715  :( M ) );
716  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
717 
718  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
719  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
720 
721  size_t i( ibegin );
722 
723  for( ; i<ipos; i+=SIMDSIZE ) {
724  y.store( i, y.load(i) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
725  }
726  for( ; remainder && i<iend; ++i ) {
727  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
728  }
729  }
730 
731  for( ; element!=end; ++element )
732  {
733  const size_t j1( element->index() );
734  const VET v1( element->value() );
735 
736  const SIMDType xmm1( set( v1 ) );
737 
738  const size_t ibegin( ( IsLower_v<MT1> )
739  ?( ( IsStrictlyLower_v<MT1> ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
740  :( 0UL ) );
741  const size_t iend( ( IsUpper_v<MT1> )
742  ?( IsStrictlyUpper_v<MT1> ? j1 : j1+1UL )
743  :( M ) );
744  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
745 
746  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
747  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
748 
749  size_t i( ibegin );
750 
751  for( ; i<ipos; i+=SIMDSIZE ) {
752  y.store( i, y.load(i) + A.load(i,j1) * xmm1 );
753  }
754  for( ; remainder && i<iend; ++i ) {
755  y[i] += A(i,j1) * v1;
756  }
757  }
758  }
760  //**********************************************************************************************
761 
762  //**Assignment to sparse vectors****************************************************************
775  template< typename VT1 > // Type of the target sparse vector
776  friend inline void assign( SparseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
777  {
779 
783 
784  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
785 
786  const ResultType tmp( serial( rhs ) );
787  assign( ~lhs, tmp );
788  }
790  //**********************************************************************************************
791 
792  //**Addition assignment to dense vectors********************************************************
805  template< typename VT1 > // Type of the target dense vector
806  friend inline void addAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
807  {
809 
810  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
811 
812  // Evaluation of the right-hand side sparse vector operand
813  RT x( serial( rhs.vec_ ) );
814  if( x.nonZeros() == 0UL ) return;
815 
816  // Evaluation of the left-hand side dense matrix operand
817  LT A( serial( rhs.mat_ ) );
818 
819  // Checking the evaluated operands
820  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
821  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
822  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
823  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
824 
825  // Performing the dense matrix-sparse vector multiplication
826  TDMatSVecMultExpr::selectAddAssignKernel( ~lhs, A, x );
827  }
829  //**********************************************************************************************
830 
831  //**Default addition assignment to dense vectors************************************************
845  template< typename VT1 // Type of the left-hand side target vector
846  , typename MT1 // Type of the left-hand side matrix operand
847  , typename VT2 > // Type of the right-hand side vector operand
848  static inline auto selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
849  -> EnableIf_t< UseDefaultKernel_v<VT1,MT1,VT2> >
850  {
851  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
852 
853  const size_t M( A.rows() );
854 
855  auto element( x.begin() );
856  const auto end( x.end() );
857 
858  for( ; element!=end; ++element )
859  {
860  const size_t index( element->index() );
861 
862  if( IsDiagonal_v<MT1> )
863  {
864  y[index] += A(index,index) * element->value();
865  }
866  else
867  {
868  const size_t ibegin( ( IsLower_v<MT1> )
869  ?( IsStrictlyLower_v<MT1> ? index+1UL : index )
870  :( 0UL ) );
871  const size_t iend( ( IsUpper_v<MT1> )
872  ?( IsStrictlyUpper_v<MT1> ? index : index+1UL )
873  :( M ) );
874  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
875 
876  for( size_t i=ibegin; i<iend; ++i ) {
877  y[i] += A(i,index) * element->value();
878  }
879  }
880  }
881  }
883  //**********************************************************************************************
884 
885  //**Optimized addition assignment to dense vectors**********************************************
899  template< typename VT1 // Type of the left-hand side target vector
900  , typename MT1 // Type of the left-hand side matrix operand
901  , typename VT2 > // Type of the right-hand side vector operand
902  static inline auto selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
903  -> EnableIf_t< UseOptimizedKernel_v<VT1,MT1,VT2> >
904  {
905  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
906 
907  const size_t M( A.rows() );
908 
909  auto element( x.begin() );
910  const auto end( x.end() );
911 
912  const size_t jpos( x.nonZeros() & size_t(-4) );
913  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
914 
915  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
916  {
917  const size_t j1( element->index() );
918  const VET v1( element->value() );
919  ++element;
920  const size_t j2( element->index() );
921  const VET v2( element->value() );
922  ++element;
923  const size_t j3( element->index() );
924  const VET v3( element->value() );
925  ++element;
926  const size_t j4( element->index() );
927  const VET v4( element->value() );
928  ++element;
929 
930  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
931 
932  const size_t ibegin( ( IsLower_v<MT1> )
933  ?( IsStrictlyLower_v<MT1> ? j1+1UL : j1 )
934  :( 0UL ) );
935  const size_t iend( ( IsUpper_v<MT1> )
936  ?( IsStrictlyUpper_v<MT1> ? j4 : j4+1UL )
937  :( M ) );
938  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
939 
940  for( size_t i=ibegin; i<iend; ++i ) {
941  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
942  }
943  }
944  for( ; element!=end; ++element )
945  {
946  const size_t j1( element->index() );
947  const VET v1( element->value() );
948 
949  const size_t ibegin( ( IsLower_v<MT1> )
950  ?( IsStrictlyLower_v<MT1> ? j1+1UL : j1 )
951  :( 0UL ) );
952  const size_t iend( ( IsUpper_v<MT1> )
953  ?( IsStrictlyUpper_v<MT1> ? j1 : j1+1UL )
954  :( M ) );
955  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
956 
957  for( size_t i=ibegin; i<iend; ++i ) {
958  y[i] += A(i,j1) * v1;
959  }
960  }
961  }
963  //**********************************************************************************************
964 
965  //**Vectorized addition assignment to dense vectors*********************************************
979  template< typename VT1 // Type of the left-hand side target vector
980  , typename MT1 // Type of the left-hand side matrix operand
981  , typename VT2 > // Type of the right-hand side vector operand
982  static inline auto selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
983  -> EnableIf_t< UseVectorizedKernel_v<VT1,MT1,VT2> >
984  {
985  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
986 
987  constexpr bool remainder( !IsPadded_v<MT1> || !IsPadded_v<VT1> );
988 
989  const size_t M( A.rows() );
990 
991  auto element( x.begin() );
992  const auto end( x.end() );
993 
994  const size_t jpos( x.nonZeros() & size_t(-4) );
995  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
996 
997  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
998  {
999  const size_t j1( element->index() );
1000  const VET v1( element->value() );
1001  ++element;
1002  const size_t j2( element->index() );
1003  const VET v2( element->value() );
1004  ++element;
1005  const size_t j3( element->index() );
1006  const VET v3( element->value() );
1007  ++element;
1008  const size_t j4( element->index() );
1009  const VET v4( element->value() );
1010  ++element;
1011 
1012  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1013 
1014  const SIMDType xmm1( set( v1 ) );
1015  const SIMDType xmm2( set( v2 ) );
1016  const SIMDType xmm3( set( v3 ) );
1017  const SIMDType xmm4( set( v4 ) );
1018 
1019  const size_t ibegin( ( IsLower_v<MT1> )
1020  ?( ( IsStrictlyLower_v<MT1> ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1021  :( 0UL ) );
1022  const size_t iend( ( IsUpper_v<MT1> )
1023  ?( IsStrictlyUpper_v<MT1> ? j4 : j4+1UL )
1024  :( M ) );
1025  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1026 
1027  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1028  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1029 
1030  size_t i( ibegin );
1031 
1032  for( ; i<ipos; i+=SIMDSIZE ) {
1033  y.store( i, y.load(i) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
1034  }
1035  for( ; remainder && i<iend; ++i ) {
1036  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1037  }
1038  }
1039  for( ; element!=end; ++element )
1040  {
1041  const size_t j1( element->index() );
1042  const VET v1( element->value() );
1043 
1044  const SIMDType xmm1( set( v1 ) );
1045 
1046  const size_t ibegin( ( IsLower_v<MT1> )
1047  ?( ( IsStrictlyLower_v<MT1> ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1048  :( 0UL ) );
1049  const size_t iend( ( IsUpper_v<MT1> )
1050  ?( IsStrictlyUpper_v<MT1> ? j1 : j1+1UL )
1051  :( M ) );
1052  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1053 
1054  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1055  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1056 
1057  size_t i( ibegin );
1058 
1059  for( ; i<ipos; i+=SIMDSIZE ) {
1060  y.store( i, y.load(i) + A.load(i,j1) * xmm1 );
1061  }
1062  for( ; remainder && i<iend; ++i ) {
1063  y[i] += A(i,j1) * v1;
1064  }
1065  }
1066  }
1068  //**********************************************************************************************
1069 
1070  //**Addition assignment to sparse vectors*******************************************************
1071  // No special implementation for the addition assignment to sparse vectors.
1072  //**********************************************************************************************
1073 
1074  //**Subtraction assignment to dense vectors*****************************************************
1087  template< typename VT1 > // Type of the target dense vector
1088  friend inline void subAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1089  {
1091 
1092  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1093 
1094  // Evaluation of the right-hand side sparse vector operand
1095  RT x( serial( rhs.vec_ ) );
1096  if( x.nonZeros() == 0UL ) return;
1097 
1098  // Evaluation of the left-hand side dense matrix operand
1099  LT A( serial( rhs.mat_ ) );
1100 
1101  // Checking the evaluated operands
1102  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1103  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1104  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1105  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1106 
1107  // Performing the dense matrix-sparse vector multiplication
1108  TDMatSVecMultExpr::selectSubAssignKernel( ~lhs, A, x );
1109  }
1111  //**********************************************************************************************
1112 
1113  //**Default subtraction assignment to dense vectors*********************************************
1127  template< typename VT1 // Type of the left-hand side target vector
1128  , typename MT1 // Type of the left-hand side matrix operand
1129  , typename VT2 > // Type of the right-hand side vector operand
1130  static inline auto selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1131  -> EnableIf_t< UseDefaultKernel_v<VT1,MT1,VT2> >
1132  {
1133  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1134 
1135  const size_t M( A.rows() );
1136 
1137  auto element( x.begin() );
1138  const auto end( x.end() );
1139 
1140  for( ; element!=end; ++element )
1141  {
1142  const size_t index( element->index() );
1143 
1144  if( IsDiagonal_v<MT1> )
1145  {
1146  y[index] -= A(index,index) * element->value();
1147  }
1148  else
1149  {
1150  const size_t ibegin( ( IsLower_v<MT1> )
1151  ?( IsStrictlyLower_v<MT1> ? index+1UL : index )
1152  :( 0UL ) );
1153  const size_t iend( ( IsUpper_v<MT1> )
1154  ?( IsStrictlyUpper_v<MT1> ? index : index+1UL )
1155  :( M ) );
1156  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1157 
1158  for( size_t i=ibegin; i<iend; ++i ) {
1159  y[i] -= A(i,index) * element->value();
1160  }
1161  }
1162  }
1163  }
1165  //**********************************************************************************************
1166 
1167  //**Optimized subtraction assignment to dense vectors*******************************************
1181  template< typename VT1 // Type of the left-hand side target vector
1182  , typename MT1 // Type of the left-hand side matrix operand
1183  , typename VT2 > // Type of the right-hand side vector operand
1184  static inline auto selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1185  -> EnableIf_t< UseOptimizedKernel_v<VT1,MT1,VT2> >
1186  {
1187  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1188 
1189  const size_t M( A.rows() );
1190 
1191  auto element( x.begin() );
1192  const auto end( x.end() );
1193 
1194  const size_t jpos( x.nonZeros() & size_t(-4) );
1195  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1196 
1197  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1198  {
1199  const size_t j1( element->index() );
1200  const VET v1( element->value() );
1201  ++element;
1202  const size_t j2( element->index() );
1203  const VET v2( element->value() );
1204  ++element;
1205  const size_t j3( element->index() );
1206  const VET v3( element->value() );
1207  ++element;
1208  const size_t j4( element->index() );
1209  const VET v4( element->value() );
1210  ++element;
1211 
1212  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1213 
1214  const size_t ibegin( ( IsLower_v<MT1> )
1215  ?( IsStrictlyLower_v<MT1> ? j1+1UL : j1 )
1216  :( 0UL ) );
1217  const size_t iend( ( IsUpper_v<MT1> )
1218  ?( IsStrictlyUpper_v<MT1> ? j4 : j4+1UL )
1219  :( M ) );
1220  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1221 
1222  for( size_t i=ibegin; i<iend; ++i ) {
1223  y[i] -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1224  }
1225  }
1226  for( ; element!=end; ++element )
1227  {
1228  const size_t j1( element->index() );
1229  const VET v1( element->value() );
1230 
1231  const size_t ibegin( ( IsLower_v<MT1> )
1232  ?( IsStrictlyLower_v<MT1> ? j1+1UL : j1 )
1233  :( 0UL ) );
1234  const size_t iend( ( IsUpper_v<MT1> )
1235  ?( IsStrictlyUpper_v<MT1> ? j1 : j1+1UL )
1236  :( M ) );
1237  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1238 
1239  for( size_t i=ibegin; i<iend; ++i ) {
1240  y[i] -= A(i,j1) * v1;
1241  }
1242  }
1243  }
1245  //**********************************************************************************************
1246 
1247  //**Vectorized subtraction assignment to dense vectors******************************************
1261  template< typename VT1 // Type of the left-hand side target vector
1262  , typename MT1 // Type of the left-hand side matrix operand
1263  , typename VT2 > // Type of the right-hand side vector operand
1264  static inline auto selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1265  -> EnableIf_t< UseVectorizedKernel_v<VT1,MT1,VT2> >
1266  {
1267  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1268 
1269  constexpr bool remainder( !IsPadded_v<MT1> || !IsPadded_v<VT1> );
1270 
1271  const size_t M( A.rows() );
1272 
1273  auto element( x.begin() );
1274  const auto end( x.end() );
1275 
1276  const size_t jpos( x.nonZeros() & size_t(-4) );
1277  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1278 
1279  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1280  {
1281  const size_t j1( element->index() );
1282  const VET v1( element->value() );
1283  ++element;
1284  const size_t j2( element->index() );
1285  const VET v2( element->value() );
1286  ++element;
1287  const size_t j3( element->index() );
1288  const VET v3( element->value() );
1289  ++element;
1290  const size_t j4( element->index() );
1291  const VET v4( element->value() );
1292  ++element;
1293 
1294  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1295 
1296  const SIMDType xmm1( set( v1 ) );
1297  const SIMDType xmm2( set( v2 ) );
1298  const SIMDType xmm3( set( v3 ) );
1299  const SIMDType xmm4( set( v4 ) );
1300 
1301  const size_t ibegin( ( IsLower_v<MT1> )
1302  ?( ( IsStrictlyLower_v<MT1> ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1303  :( 0UL ) );
1304  const size_t iend( ( IsUpper_v<MT1> )
1305  ?( IsStrictlyUpper_v<MT1> ? j4 : j4+1UL )
1306  :( M ) );
1307  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1308 
1309  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1310  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1311 
1312  size_t i( ibegin );
1313 
1314  for( ; i<ipos; i+=SIMDSIZE ) {
1315  y.store( i, y.load(i) - A.load(i,j1) * xmm1 - A.load(i,j2) * xmm2 - A.load(i,j3) * xmm3 - A.load(i,j4) * xmm4 );
1316  }
1317  for( ; remainder && i<iend; ++i ) {
1318  y[i] -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1319  }
1320  }
1321  for( ; element!=end; ++element )
1322  {
1323  const size_t j1( element->index() );
1324  const VET v1( element->value() );
1325 
1326  const SIMDType xmm1( set( v1 ) );
1327 
1328  const size_t ibegin( ( IsLower_v<MT1> )
1329  ?( ( IsStrictlyLower_v<MT1> ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1330  :( 0UL ) );
1331  const size_t iend( ( IsUpper_v<MT1> )
1332  ?( IsStrictlyUpper_v<MT1> ? j1 : j1+1UL )
1333  :( M ) );
1334  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1335 
1336  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1337  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1338 
1339  size_t i( ibegin );
1340 
1341  for( ; i<ipos; i+=SIMDSIZE ) {
1342  y.store( i, y.load(i) - A.load(i,j1) * xmm1 );
1343  }
1344  for( ; remainder && i<iend; ++i ) {
1345  y[i] -= A(i,j1) * v1;
1346  }
1347  }
1348  }
1350  //**********************************************************************************************
1351 
1352  //**Subtraction assignment to sparse vectors****************************************************
1353  // No special implementation for the subtraction assignment to sparse vectors.
1354  //**********************************************************************************************
1355 
1356  //**Multiplication assignment to dense vectors**************************************************
1369  template< typename VT1 > // Type of the target dense vector
1370  friend inline void multAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1371  {
1373 
1377 
1378  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1379 
1380  const ResultType tmp( serial( rhs ) );
1381  multAssign( ~lhs, tmp );
1382  }
1384  //**********************************************************************************************
1385 
1386  //**Multiplication assignment to sparse vectors*************************************************
1387  // No special implementation for the multiplication assignment to sparse vectors.
1388  //**********************************************************************************************
1389 
1390  //**Division assignment to dense vectors********************************************************
1403  template< typename VT1 > // Type of the target dense vector
1404  friend inline void divAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1405  {
1407 
1411 
1412  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1413 
1414  const ResultType tmp( serial( rhs ) );
1415  divAssign( ~lhs, tmp );
1416  }
1418  //**********************************************************************************************
1419 
1420  //**Division assignment to sparse vectors*******************************************************
1421  // No special implementation for the division assignment to sparse vectors.
1422  //**********************************************************************************************
1423 
1424  //**SMP assignment to dense vectors*************************************************************
1439  template< typename VT1 > // Type of the target dense vector
1440  friend inline auto smpAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1441  -> EnableIf_t< UseSMPAssign_v<VT1> >
1442  {
1444 
1445  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1446 
1447  // Evaluation of the right-hand side sparse vector operand
1448  RT x( rhs.vec_ );
1449  if( x.nonZeros() == 0UL ) {
1450  reset( ~lhs );
1451  return;
1452  }
1453 
1454  // Evaluation of the left-hand side dense matrix operand
1455  LT A( rhs.mat_ );
1456 
1457  // Checking the evaluated operands
1458  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1459  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1460  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1461  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1462 
1463  // Performing the dense matrix-sparse vector multiplication
1464  smpAssign( ~lhs, A * x );
1465  }
1467  //**********************************************************************************************
1468 
1469  //**SMP assignment to sparse vectors************************************************************
1484  template< typename VT1 > // Type of the target sparse vector
1485  friend inline auto smpAssign( SparseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1486  -> EnableIf_t< UseSMPAssign_v<VT1> >
1487  {
1489 
1493 
1494  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1495 
1496  const ResultType tmp( rhs );
1497  smpAssign( ~lhs, tmp );
1498  }
1500  //**********************************************************************************************
1501 
1502  //**SMP addition assignment to dense vectors****************************************************
1517  template< typename VT1 > // Type of the target dense vector
1518  friend inline auto smpAddAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1519  -> EnableIf_t< UseSMPAssign_v<VT1> >
1520  {
1522 
1523  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1524 
1525  // Evaluation of the right-hand side sparse vector operand
1526  RT x( rhs.vec_ );
1527  if( x.nonZeros() == 0UL ) return;
1528 
1529  // Evaluation of the left-hand side dense matrix operand
1530  LT A( rhs.mat_ );
1531 
1532  // Checking the evaluated operands
1533  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1534  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1535  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1536  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1537 
1538  // Performing the dense matrix-sparse vector multiplication
1539  smpAddAssign( ~lhs, A * x );
1540  }
1542  //**********************************************************************************************
1543 
1544  //**SMP addition assignment to sparse vectors***************************************************
1545  // No special implementation for the SMP addition assignment to sparse vectors.
1546  //**********************************************************************************************
1547 
1548  //**SMP subtraction assignment to dense vectors*************************************************
1563  template< typename VT1 > // Type of the target dense vector
1564  friend inline auto smpSubAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1565  -> EnableIf_t< UseSMPAssign_v<VT1> >
1566  {
1568 
1569  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1570 
1571  // Evaluation of the right-hand side sparse vector operand
1572  RT x( rhs.vec_ );
1573  if( x.nonZeros() == 0UL ) return;
1574 
1575  // Evaluation of the left-hand side dense matrix operand
1576  LT A( rhs.mat_ );
1577 
1578  // Checking the evaluated operands
1579  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1580  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1581  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1582  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1583 
1584  // Performing the dense matrix-sparse vector multiplication
1585  smpSubAssign( ~lhs, A * x );
1586  }
1588  //**********************************************************************************************
1589 
1590  //**SMP subtraction assignment to sparse vectors************************************************
1591  // No special implementation for the SMP subtraction assignment to sparse vectors.
1592  //**********************************************************************************************
1593 
1594  //**SMP multiplication assignment to dense vectors**********************************************
1609  template< typename VT1 > // Type of the target dense vector
1610  friend inline auto smpMultAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1611  -> EnableIf_t< UseSMPAssign_v<VT1> >
1612  {
1614 
1618 
1619  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1620 
1621  const ResultType tmp( rhs );
1622  smpMultAssign( ~lhs, tmp );
1623  }
1625  //**********************************************************************************************
1626 
1627  //**SMP multiplication assignment to sparse vectors*********************************************
1628  // No special implementation for the SMP multiplication assignment to sparse vectors.
1629  //**********************************************************************************************
1630 
1631  //**SMP division assignment to dense vectors****************************************************
1646  template< typename VT1 > // Type of the target dense vector
1647  friend inline auto smpDivAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1648  -> EnableIf_t< UseSMPAssign_v<VT1> >
1649  {
1651 
1655 
1656  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1657 
1658  const ResultType tmp( rhs );
1659  smpDivAssign( ~lhs, tmp );
1660  }
1662  //**********************************************************************************************
1663 
1664  //**SMP division assignment to sparse vectors***************************************************
1665  // No special implementation for the SMP division assignment to sparse vectors.
1666  //**********************************************************************************************
1667 
1668  //**Compile time checks*************************************************************************
1677  //**********************************************************************************************
1678 };
1679 //*************************************************************************************************
1680 
1681 
1682 
1683 
1684 //=================================================================================================
1685 //
1686 // GLOBAL BINARY ARITHMETIC OPERATORS
1687 //
1688 //=================================================================================================
1689 
1690 //*************************************************************************************************
1703 template< typename MT // Type of the left-hand side dense matrix
1704  , typename VT // Type of the right-hand side sparse vector
1705  , DisableIf_t< IsZero_v<VT> >* = nullptr >
1706 inline const TDMatSVecMultExpr<MT,VT>
1707  tdmatsvecmult( const DenseMatrix<MT,true>& mat, const SparseVector<VT,false>& vec )
1708 {
1710 
1711  BLAZE_INTERNAL_ASSERT( (~mat).columns() == (~vec).size(), "Invalid matrix and vector sizes" );
1712 
1713  return TDMatSVecMultExpr<MT,VT>( ~mat, ~vec );
1714 }
1716 //*************************************************************************************************
1717 
1718 
1719 //*************************************************************************************************
1732 template< typename MT // Type of the left-hand side dense matrix
1733  , typename VT // Type of the right-hand side sparse vector
1734  , EnableIf_t< IsZero_v<VT> >* = nullptr >
1735 inline decltype(auto)
1736  tdmatsvecmult( const DenseMatrix<MT,true>& mat, const SparseVector<VT,false>& vec )
1737 {
1739 
1740  UNUSED_PARAMETER( vec );
1741 
1742  BLAZE_INTERNAL_ASSERT( (~mat).columns() == (~vec).size(), "Invalid matrix and vector sizes" );
1743 
1744  using ReturnType = const MultTrait_t< ResultType_t<MT>, ResultType_t<VT> >;
1745 
1748 
1749  return ReturnType( (~mat).rows() );
1750 }
1752 //*************************************************************************************************
1753 
1754 
1755 //*************************************************************************************************
1787 template< typename MT // Type of the left-hand side dense matrix
1788  , typename VT > // Type of the right-hand side sparse vector
1789 inline decltype(auto)
1790  operator*( const DenseMatrix<MT,true>& mat, const SparseVector<VT,false>& vec )
1791 {
1793 
1795 
1796  if( (~mat).columns() != (~vec).size() ) {
1797  BLAZE_THROW_INVALID_ARGUMENT( "Matrix and vector sizes do not match" );
1798  }
1799 
1800  return tdmatsvecmult( ~mat, ~vec );
1801 }
1802 //*************************************************************************************************
1803 
1804 
1805 
1806 
1807 //=================================================================================================
1808 //
1809 // ISALIGNED SPECIALIZATIONS
1810 //
1811 //=================================================================================================
1812 
1813 //*************************************************************************************************
1815 template< typename MT, typename VT >
1816 struct IsAligned< TDMatSVecMultExpr<MT,VT> >
1817  : public IsAligned<MT>
1818 {};
1820 //*************************************************************************************************
1821 
1822 } // namespace blaze
1823 
1824 #endif
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:329
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Header file for auxiliary alias declarations.
Header file for the blaze::checked and blaze::unchecked instances.
RightOperand vec_
Right-hand side sparse vector of the multiplication expression.
Definition: TDMatSVecMultExpr.h:368
SIMDTrait_t< ElementType > SIMDType
Resulting SIMD element type.
Definition: TDMatSVecMultExpr.h:194
Header file for the UNUSED_PARAMETER function template.
Header file for basic type definitions.
typename If< Condition, T1, T2 >::Type If_t
Auxiliary alias declaration for the If class template.The If_t alias declaration provides a convenien...
Definition: If.h:109
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatSVecMultExpr.h:350
ElementType_t< MRT > MET
Element type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:115
typename T::ResultType ResultType_t
Alias declaration for nested ResultType type definitions.The ResultType_t alias declaration provides ...
Definition: Aliases.h:390
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a column dense or sparse vector type...
Definition: ColumnVector.h:61
Header file for the serial shim.
Header file for the IsDiagonal type trait.
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatSVecMultExpr.h:196
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:372
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:591
static constexpr bool smpAssignable
Compilation flag for SMP assignments.
Definition: CompressedMatrix.h:3113
constexpr Unchecked unchecked
Global Unchecked instance.The blaze::unchecked instance is an optional token for the creation of view...
Definition: Check.h:138
Constraint on the data type.
typename SIMDTrait< T >::Type SIMDTrait_t
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_t alias declaration provid...
Definition: SIMDTrait.h:315
Header file for the DenseVector base class.
Header file for the Computation base class.
Header file for the reset shim.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
constexpr void UNUSED_PARAMETER(const Args &...)
Suppression of unused parameter warnings.
Definition: Unused.h:81
static constexpr bool smpAssignable
Compilation switch for the expression template assignment strategy.
Definition: TDMatSVecMultExpr.h:220
constexpr size_t columns(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of columns of the matrix.
Definition: Matrix.h:514
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:80
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.The ElementType_t alias declaration provide...
Definition: Aliases.h:170
auto smpDivAssign(Vector< VT1, TF1 > &lhs, const Vector< VT2, TF2 > &rhs) -> EnableIf_t< IsDenseVector_v< VT1 > >
Default implementation of the SMP division assignment of a vector to a dense vector.
Definition: DenseVector.h:220
Constraint on the transpose flag of vector types.
ReturnType at(size_t index) const
Checked access to the vector elements.
Definition: TDMatSVecMultExpr.h:283
Constraint on the data type.
typename EnableIf< Condition, T >::Type EnableIf_t
Auxiliary type for the EnableIf class template.The EnableIf_t alias declaration provides a convenient...
Definition: EnableIf.h:138
Header file for the DisableIf class template.
CompositeType_t< VT > VCT
Composite type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:118
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:58
Header file for the If class template.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
ElementType_t< VRT > VET
Element type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:116
ResultType_t< VT > VRT
Result type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:114
If_t< evaluateVector, const VRT, CompositeType_t< VT > > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: TDMatSVecMultExpr.h:208
#define BLAZE_CONSTRAINT_MUST_BE_ZERO_TYPE(T)
Constraint on the data type.In case the given data type T is not a zero vector or matrix type...
Definition: Zero.h:61
LeftOperand mat_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatSVecMultExpr.h:367
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
static constexpr bool evaluateMatrix
Compilation switch for the composite type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:123
MultTrait_t< MRT, VRT > ResultType
Result type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:191
If_t< IsExpression_v< MT >, const MT, const MT &> LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:199
Header file for all SIMD functionality.
Base class for N-dimensional dense vectors.The DenseVector class is a base class for all arbitrarily ...
Definition: DenseVector.h:76
Header file for the IsLower type trait.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional vector type...
Definition: SparseVector.h:61
Header file for the IsAligned type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_MATMATMULTEXPR_TYPE(T)
Constraint on the data type.In case the given data type T is a matrix/matrix multiplication expressio...
Definition: MatMatMultExpr.h:83
LeftOperand leftOperand() const noexcept
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatSVecMultExpr.h:306
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATVECMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/vector ...
Definition: MatVecMultExpr.h:104
Constraint on the data type.
Header file for the exception macros of the math module.
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:438
ReturnType operator[](size_t index) const
Subscript operator for the direct access to the vector elements.
Definition: TDMatSVecMultExpr.h:249
Constraint on the data type.
Header file for all forward declarations for expression class templates.
Constraint on the data type.
ResultType_t< MT > MRT
Result type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:113
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
Header file for the IsPadded type trait.
typename MultTrait< T1, T2 >::Type MultTrait_t
Auxiliary alias declaration for the MultTrait class template.The MultTrait_t alias declaration provid...
Definition: MultTrait.h:240
RightOperand rightOperand() const noexcept
Returns the right-hand side sparse vector operand.
Definition: TDMatSVecMultExpr.h:316
Expression object for transpose dense matrix-sparse vector multiplications.The TDMatSVecMultExpr clas...
Definition: Forward.h:162
CompositeType_t< MT > MCT
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:117
Header file for the IsSIMDCombinable type trait.
Header file for the HasSIMDMult type trait.
typename T::TransposeType TransposeType_t
Alias declaration for nested TransposeType type definitions.The TransposeType_t alias declaration pro...
Definition: Aliases.h:470
Header file for run time assertion macros.
typename T::CompositeType CompositeType_t
Alias declaration for nested CompositeType type definitions.The CompositeType_t alias declaration pro...
Definition: Aliases.h:90
size_t size() const noexcept
Returns the current size/dimension of the vector.
Definition: TDMatSVecMultExpr.h:296
auto smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:131
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:133
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TDMatSVecMultExpr.h:328
Header file for the IsZero type trait.
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:295
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:94
constexpr size_t size(const Matrix< MT, SO > &matrix) noexcept
Returns the total number of elements of the matrix.
Definition: Matrix.h:530
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
Constraints on the storage order of matrix types.
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:808
TDMatSVecMultExpr(const MT &mat, const VT &vec) noexcept
Constructor for the TDMatSVecMultExpr class.
Definition: TDMatSVecMultExpr.h:235
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
constexpr size_t rows(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of rows of the matrix.
Definition: Matrix.h:498
TransposeType_t< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:192
const Type & ReturnType
Return type for expression template evaluations.
Definition: CompressedMatrix.h:3081
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional vector type...
Definition: DenseVector.h:61
static constexpr bool simdEnabled
Compilation switch for the expression template evaluation strategy.
Definition: TDMatSVecMultExpr.h:213
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TDMatSVecMultExpr.h:340
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TDMatSVecMultExpr.h:360
Header file for the IsComputation type trait class.
auto smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:162
If_t< IsExpression_v< VT >, const VT, const VT &> RightOperand
Composite type of the right-hand side dense vector expression.
Definition: TDMatSVecMultExpr.h:202
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:195
ElementType_t< ResultType > ElementType
Resulting element type.
Definition: TDMatSVecMultExpr.h:193
Base class for sparse vectors.The SparseVector class is a base class for all arbitrarily sized (N-dim...
Definition: Forward.h:138
static constexpr bool evaluateVector
Compilation switch for the composite type of the right-hand side dense vector expression.
Definition: TDMatSVecMultExpr.h:128
Constraint on the data type.
Header file for the IsUpper type trait.
If_t< evaluateMatrix, const MRT, MCT > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatSVecMultExpr.h:205
Header file for the MatVecMultExpr base class.
Constraint on the data type.
static constexpr size_t SIMDSIZE
The number of elements packed within a single SIMD element.
Definition: TDMatSVecMultExpr.h:226
Header file for the IsResizable type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_ZERO_TYPE(T)
Constraint on the data type.In case the given data type T is a zero vector or matrix type...
Definition: Zero.h:81
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
auto smpMultAssign(Vector< VT1, TF1 > &lhs, const Vector< VT2, TF2 > &rhs) -> EnableIf_t< IsDenseVector_v< VT1 > >
Default implementation of the SMP multiplication assignment of a vector to a dense vector...
Definition: DenseVector.h:191
Header file for the IsExpression type trait class.
Header file for the function trace functionality.