TDMatSVecMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATSVECMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATSVECMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
51 #include <blaze/math/Exception.h>
56 #include <blaze/math/shims/Reset.h>
58 #include <blaze/math/SIMD.h>
81 #include <blaze/util/Assert.h>
83 #include <blaze/util/DisableIf.h>
84 #include <blaze/util/EnableIf.h>
87 #include <blaze/util/mpl/If.h>
88 #include <blaze/util/Types.h>
91 
92 
93 namespace blaze {
94 
95 //=================================================================================================
96 //
97 // CLASS TDMATSVECMULTEXPR
98 //
99 //=================================================================================================
100 
101 //*************************************************************************************************
108 template< typename MT // Type of the left-hand side dense matrix
109  , typename VT > // Type of the right-hand side sparse vector
110 class TDMatSVecMultExpr : public DenseVector< TDMatSVecMultExpr<MT,VT>, false >
111  , private MatVecMultExpr
112  , private Computation
113 {
114  private:
115  //**Type definitions****************************************************************************
122  //**********************************************************************************************
123 
124  //**********************************************************************************************
126  enum : bool { evaluateMatrix = RequiresEvaluation<MT>::value };
127  //**********************************************************************************************
128 
129  //**********************************************************************************************
131  enum : bool { evaluateVector = IsComputation<VT>::value || RequiresEvaluation<VT>::value };
132  //**********************************************************************************************
133 
134  //**********************************************************************************************
136 
140  template< typename T1 >
141  struct UseSMPAssign {
142  enum : bool { value = ( evaluateMatrix || evaluateVector ) };
143  };
145  //**********************************************************************************************
146 
147  //**********************************************************************************************
149 
153  template< typename T1, typename T2, typename T3 >
154  struct UseVectorizedKernel {
155  enum : bool { value = useOptimizedKernels &&
156  !IsDiagonal<T2>::value &&
157  T1::simdEnabled && T2::simdEnabled &&
158  AreSIMDCombinable< ElementType_<T1>
159  , ElementType_<T2>
160  , ElementType_<T3> >::value &&
161  HasSIMDAdd< ElementType_<T2>, ElementType_<T3> >::value &&
162  HasSIMDMult< ElementType_<T2>, ElementType_<T3> >::value };
163  };
165  //**********************************************************************************************
166 
167  //**********************************************************************************************
169 
173  template< typename T1, typename T2, typename T3 >
174  struct UseOptimizedKernel {
175  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
176  !IsDiagonal<T2>::value &&
177  !IsResizable< ElementType_<T1> >::value &&
178  !IsResizable<VET>::value };
179  };
181  //**********************************************************************************************
182 
183  //**********************************************************************************************
185 
188  template< typename T1, typename T2, typename T3 >
189  struct UseDefaultKernel {
190  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
191  !UseOptimizedKernel<T1,T2,T3>::value };
192  };
194  //**********************************************************************************************
195 
196  public:
197  //**Type definitions****************************************************************************
203  typedef const ElementType ReturnType;
204  typedef const ResultType CompositeType;
205 
207  typedef If_< IsExpression<MT>, const MT, const MT& > LeftOperand;
208 
210  typedef If_< IsExpression<VT>, const VT, const VT& > RightOperand;
211 
214 
217  //**********************************************************************************************
218 
219  //**Compilation flags***************************************************************************
221  enum : bool { simdEnabled = !IsDiagonal<MT>::value &&
222  MT::simdEnabled &&
225 
227  enum : bool { smpAssignable = !evaluateMatrix && MT::smpAssignable &&
228  !evaluateVector && VT::smpAssignable };
229  //**********************************************************************************************
230 
231  //**SIMD properties*****************************************************************************
233  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
234  //**********************************************************************************************
235 
236  //**Constructor*********************************************************************************
242  explicit inline TDMatSVecMultExpr( const MT& mat, const VT& vec ) noexcept
243  : mat_( mat ) // Left-hand side dense matrix of the multiplication expression
244  , vec_( vec ) // Right-hand side sparse vector of the multiplication expression
245  {
246  BLAZE_INTERNAL_ASSERT( mat_.columns() == vec_.size(), "Invalid matrix and vector sizes" );
247  }
248  //**********************************************************************************************
249 
250  //**Subscript operator**************************************************************************
256  inline ReturnType operator[]( size_t index ) const {
257  BLAZE_INTERNAL_ASSERT( index < mat_.rows(), "Invalid vector access index" );
258 
260  {
261  return mat_(index,index) * vec_[index];
262  }
263  else if( IsLower<MT>::value )
264  {
265  const size_t n( IsStrictlyLower<MT>::value ? index : index+1UL );
266  return subvector( row( mat_, index ), 0UL, n ) * subvector( vec_, 0UL, n );
267  }
268  else if( IsUpper<MT>::value )
269  {
270  const size_t begin( IsStrictlyUpper<MT>::value ? index+1UL : index );
271  const size_t n ( mat_.columns() - begin );
272  return subvector( row( mat_, index ), begin, n ) * subvector( vec_, begin, n );
273  }
274  else
275  {
276  return row( mat_, index ) * vec_;
277  }
278  }
279  //**********************************************************************************************
280 
281  //**At function*********************************************************************************
288  inline ReturnType at( size_t index ) const {
289  if( index >= mat_.rows() ) {
290  BLAZE_THROW_OUT_OF_RANGE( "Invalid vector access index" );
291  }
292  return (*this)[index];
293  }
294  //**********************************************************************************************
295 
296  //**Size function*******************************************************************************
301  inline size_t size() const noexcept {
302  return mat_.rows();
303  }
304  //**********************************************************************************************
305 
306  //**Left operand access*************************************************************************
311  inline LeftOperand leftOperand() const noexcept {
312  return mat_;
313  }
314  //**********************************************************************************************
315 
316  //**Right operand access************************************************************************
321  inline RightOperand rightOperand() const noexcept {
322  return vec_;
323  }
324  //**********************************************************************************************
325 
326  //**********************************************************************************************
332  template< typename T >
333  inline bool canAlias( const T* alias ) const noexcept {
334  return mat_.isAliased( alias ) || vec_.isAliased( alias );
335  }
336  //**********************************************************************************************
337 
338  //**********************************************************************************************
344  template< typename T >
345  inline bool isAliased( const T* alias ) const noexcept {
346  return mat_.isAliased( alias ) || vec_.isAliased( alias );
347  }
348  //**********************************************************************************************
349 
350  //**********************************************************************************************
355  inline bool isAligned() const noexcept {
356  return mat_.isAligned();
357  }
358  //**********************************************************************************************
359 
360  //**********************************************************************************************
365  inline bool canSMPAssign() const noexcept {
366  return ( size() > SMP_TDMATSVECMULT_THRESHOLD );
367  }
368  //**********************************************************************************************
369 
370  private:
371  //**Member variables****************************************************************************
372  LeftOperand mat_;
373  RightOperand vec_;
374  //**********************************************************************************************
375 
376  //**Assignment to dense vectors*****************************************************************
389  template< typename VT1 > // Type of the target dense vector
390  friend inline void assign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
391  {
393 
394  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
395 
396  // Evaluation of the right-hand side sparse vector operand
397  RT x( serial( rhs.vec_ ) );
398  if( x.nonZeros() == 0UL ) {
399  reset( ~lhs );
400  return;
401  }
402 
403  // Evaluation of the left-hand side dense matrix operand
404  LT A( serial( rhs.mat_ ) );
405 
406  // Checking the evaluated operands
407  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
408  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
409  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
410  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
411 
412  // Performing the dense matrix-sparse vector multiplication
413  TDMatSVecMultExpr::selectAssignKernel( ~lhs, A, x );
414  }
416  //**********************************************************************************************
417 
418  //**Default assignment to dense vectors*********************************************************
432  template< typename VT1 // Type of the left-hand side target vector
433  , typename MT1 // Type of the left-hand side matrix operand
434  , typename VT2 > // Type of the right-hand side vector operand
436  selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
437  {
439 
440  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
441 
442  const size_t M( A.rows() );
443 
444  ConstIterator element( x.begin() );
445  const ConstIterator end( x.end() );
446 
447  size_t last( 0UL );
448 
449  if( IsLower<MT1>::value ) {
450  const size_t iend( IsStrictlyLower<MT1>::value ? element->index()+1UL : element->index() );
451  for( size_t i=0UL; i<iend; ++i )
452  reset( y[i] );
453  }
454 
455  for( ; element!=end; ++element )
456  {
457  const size_t index( element->index() );
458 
459  if( IsDiagonal<MT1>::value )
460  {
461  for( size_t i=last; i<index; ++i )
462  reset( y[i] );
463 
464  y[index] = A(index,index) * element->value();
465  last = index + 1UL;
466  }
467  else
468  {
469  const size_t ibegin( ( IsLower<MT1>::value )
470  ?( IsStrictlyLower<MT1>::value ? index+1UL : index )
471  :( 0UL ) );
472  const size_t iend( ( IsUpper<MT1>::value )
473  ?( IsStrictlyUpper<MT1>::value ? index : index+1UL )
474  :( M ) );
475  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
476 
477  for( size_t i=ibegin; i<last; ++i ) {
478  y[i] += A(i,index) * element->value();
479  }
480  for( size_t i=last; i<iend; ++i ) {
481  y[i] = A(i,index) * element->value();
482  }
483 
484  last = iend;
485  }
486  }
487 
488  if( IsUpper<MT1>::value ) {
489  for( size_t i=last; i<M; ++i )
490  reset( y[i] );
491  }
492  }
494  //**********************************************************************************************
495 
496  //**Optimized assignment to dense vectors*******************************************************
510  template< typename VT1 // Type of the left-hand side target vector
511  , typename MT1 // Type of the left-hand side matrix operand
512  , typename VT2 > // Type of the right-hand side vector operand
513  static inline EnableIf_< UseOptimizedKernel<VT1,MT1,VT2> >
514  selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
515  {
516  typedef ConstIterator_< RemoveReference_<RT> > ConstIterator;
517 
518  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
519 
520  const size_t M( A.rows() );
521 
522  ConstIterator element( x.begin() );
523  const ConstIterator end( x.end() );
524 
525  const size_t jpos( x.nonZeros() & size_t(-4) );
526  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
527 
528  if( jpos > 3UL )
529  {
530  const size_t j1( element->index() );
531  const VET v1( element->value() );
532  ++element;
533  const size_t j2( element->index() );
534  const VET v2( element->value() );
535  ++element;
536  const size_t j3( element->index() );
537  const VET v3( element->value() );
538  ++element;
539  const size_t j4( element->index() );
540  const VET v4( element->value() );
541  ++element;
542 
543  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
544 
545  for( size_t i=0UL; i<M; ++i ) {
546  y[i] = A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
547  }
548  }
549  else
550  {
551  const size_t j1( element->index() );
552  const VET v1( element->value() );
553  ++element;
554 
555  for( size_t i=0UL; i<M; ++i ) {
556  y[i] = A(i,j1) * v1;
557  }
558  }
559 
560  for( size_t j=(jpos>3UL)?(4UL):(1UL); (j+4UL)<=jpos; j+=4UL )
561  {
562  const size_t j1( element->index() );
563  const VET v1( element->value() );
564  ++element;
565  const size_t j2( element->index() );
566  const VET v2( element->value() );
567  ++element;
568  const size_t j3( element->index() );
569  const VET v3( element->value() );
570  ++element;
571  const size_t j4( element->index() );
572  const VET v4( element->value() );
573  ++element;
574 
575  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
576 
577  const size_t ibegin( ( IsLower<MT1>::value )
578  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
579  :( 0UL ) );
580  const size_t iend( ( IsUpper<MT1>::value )
581  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
582  :( M ) );
583  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
584 
585  for( size_t i=ibegin; i<iend; ++i ) {
586  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
587  }
588  }
589  for( ; element!=end; ++element )
590  {
591  const size_t j1( element->index() );
592  const VET v1( element->value() );
593 
594  const size_t ibegin( ( IsLower<MT1>::value )
595  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
596  :( 0UL ) );
597  const size_t iend( ( IsUpper<MT1>::value )
598  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
599  :( M ) );
600  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
601 
602  for( size_t i=ibegin; i<iend; ++i ) {
603  y[i] += A(i,j1) * v1;
604  }
605  }
606  }
608  //**********************************************************************************************
609 
610  //**Vectorized assignment to dense vectors******************************************************
624  template< typename VT1 // Type of the left-hand side target vector
625  , typename MT1 // Type of the left-hand side matrix operand
626  , typename VT2 > // Type of the right-hand side vector operand
627  static inline EnableIf_< UseVectorizedKernel<VT1,MT1,VT2> >
628  selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
629  {
630  typedef ConstIterator_< RemoveReference_<RT> > ConstIterator;
631 
632  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
633 
634  const size_t M( A.rows() );
635 
636  const bool remainder( !IsPadded<MT1>::value || !IsPadded<VT1>::value );
637 
638  ConstIterator element( x.begin() );
639  const ConstIterator end( x.end() );
640 
641  const size_t jpos( x.nonZeros() & size_t(-4) );
642  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
643 
644  if( jpos > 3UL )
645  {
646  const size_t j1( element->index() );
647  const VET v1( element->value() );
648  ++element;
649  const size_t j2( element->index() );
650  const VET v2( element->value() );
651  ++element;
652  const size_t j3( element->index() );
653  const VET v3( element->value() );
654  ++element;
655  const size_t j4( element->index() );
656  const VET v4( element->value() );
657  ++element;
658 
659  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
660 
661  const SIMDType xmm1( set( v1 ) );
662  const SIMDType xmm2( set( v2 ) );
663  const SIMDType xmm3( set( v3 ) );
664  const SIMDType xmm4( set( v4 ) );
665 
666  const size_t ipos( remainder ? ( M & size_t(-SIMDSIZE) ) : M );
667  BLAZE_INTERNAL_ASSERT( !remainder || ( M - ( M % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
668 
669  size_t i( 0UL );
670 
671  for( ; i<ipos; i+=SIMDSIZE ) {
672  y.store( i, A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
673  }
674  for( ; remainder && i<M; ++i ) {
675  y[i] = A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
676  }
677  }
678  else
679  {
680  const size_t j1( element->index() );
681  const VET v1( element->value() );
682  ++element;
683 
684  const SIMDType xmm1( set( v1 ) );
685 
686  const size_t ipos( remainder ? ( M & size_t(-SIMDSIZE) ) : M );
687  BLAZE_INTERNAL_ASSERT( !remainder || ( M - ( M % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
688 
689  size_t i( 0UL );
690 
691  for( ; i<ipos; i+=SIMDSIZE ) {
692  y.store( i, A.load(i,j1) * xmm1 );
693  }
694  for( ; remainder && i<M; ++i ) {
695  y[i] = A(i,j1) * v1;
696  }
697  }
698 
699  for( size_t j=(jpos>3UL)?(4UL):(1UL); (j+4UL)<=jpos; j+=4UL )
700  {
701  const size_t j1( element->index() );
702  const VET v1( element->value() );
703  ++element;
704  const size_t j2( element->index() );
705  const VET v2( element->value() );
706  ++element;
707  const size_t j3( element->index() );
708  const VET v3( element->value() );
709  ++element;
710  const size_t j4( element->index() );
711  const VET v4( element->value() );
712  ++element;
713 
714  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
715 
716  const SIMDType xmm1( set( v1 ) );
717  const SIMDType xmm2( set( v2 ) );
718  const SIMDType xmm3( set( v3 ) );
719  const SIMDType xmm4( set( v4 ) );
720 
721  const size_t ibegin( ( IsLower<MT1>::value )
722  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
723  :( 0UL ) );
724  const size_t iend( ( IsUpper<MT1>::value )
725  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
726  :( M ) );
727  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
728 
729  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
730  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
731 
732  size_t i( ibegin );
733 
734  for( ; i<ipos; i+=SIMDSIZE ) {
735  y.store( i, y.load(i) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
736  }
737  for( ; remainder && i<iend; ++i ) {
738  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
739  }
740  }
741 
742  for( ; element!=end; ++element )
743  {
744  const size_t j1( element->index() );
745  const VET v1( element->value() );
746 
747  const SIMDType xmm1( set( v1 ) );
748 
749  const size_t ibegin( ( IsLower<MT1>::value )
750  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
751  :( 0UL ) );
752  const size_t iend( ( IsUpper<MT1>::value )
753  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
754  :( M ) );
755  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
756 
757  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
758  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
759 
760  size_t i( ibegin );
761 
762  for( ; i<ipos; i+=SIMDSIZE ) {
763  y.store( i, y.load(i) + A.load(i,j1) * xmm1 );
764  }
765  for( ; remainder && i<iend; ++i ) {
766  y[i] += A(i,j1) * v1;
767  }
768  }
769  }
771  //**********************************************************************************************
772 
773  //**Assignment to sparse vectors****************************************************************
786  template< typename VT1 > // Type of the target sparse vector
787  friend inline void assign( SparseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
788  {
790 
793  BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE( CompositeType_<ResultType> );
794 
795  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
796 
797  const ResultType tmp( serial( rhs ) );
798  assign( ~lhs, tmp );
799  }
801  //**********************************************************************************************
802 
803  //**Addition assignment to dense vectors********************************************************
816  template< typename VT1 > // Type of the target dense vector
817  friend inline void addAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
818  {
820 
821  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
822 
823  // Evaluation of the right-hand side sparse vector operand
824  RT x( serial( rhs.vec_ ) );
825  if( x.nonZeros() == 0UL ) return;
826 
827  // Evaluation of the left-hand side dense matrix operand
828  LT A( serial( rhs.mat_ ) );
829 
830  // Checking the evaluated operands
831  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
832  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
833  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
834  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
835 
836  // Performing the dense matrix-sparse vector multiplication
837  TDMatSVecMultExpr::selectAddAssignKernel( ~lhs, A, x );
838  }
840  //**********************************************************************************************
841 
842  //**Default addition assignment to dense vectors************************************************
856  template< typename VT1 // Type of the left-hand side target vector
857  , typename MT1 // Type of the left-hand side matrix operand
858  , typename VT2 > // Type of the right-hand side vector operand
859  static inline EnableIf_< UseDefaultKernel<VT1,MT1,VT2> >
860  selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
861  {
862  typedef ConstIterator_< RemoveReference_<RT> > ConstIterator;
863 
864  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
865 
866  const size_t M( A.rows() );
867 
868  ConstIterator element( x.begin() );
869  const ConstIterator end( x.end() );
870 
871  for( ; element!=end; ++element )
872  {
873  const size_t index( element->index() );
874 
875  if( IsDiagonal<MT1>::value )
876  {
877  y[index] += A(index,index) * element->value();
878  }
879  else
880  {
881  const size_t ibegin( ( IsLower<MT1>::value )
882  ?( IsStrictlyLower<MT1>::value ? index+1UL : index )
883  :( 0UL ) );
884  const size_t iend( ( IsUpper<MT1>::value )
885  ?( IsStrictlyUpper<MT1>::value ? index : index+1UL )
886  :( M ) );
887  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
888 
889  for( size_t i=ibegin; i<iend; ++i ) {
890  y[i] += A(i,index) * element->value();
891  }
892  }
893  }
894  }
896  //**********************************************************************************************
897 
898  //**Optimized addition assignment to dense vectors**********************************************
912  template< typename VT1 // Type of the left-hand side target vector
913  , typename MT1 // Type of the left-hand side matrix operand
914  , typename VT2 > // Type of the right-hand side vector operand
915  static inline EnableIf_< UseOptimizedKernel<VT1,MT1,VT2> >
916  selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
917  {
918  typedef ConstIterator_< RemoveReference_<RT> > ConstIterator;
919 
920  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
921 
922  const size_t M( A.rows() );
923 
924  ConstIterator element( x.begin() );
925  const ConstIterator end( x.end() );
926 
927  const size_t jpos( x.nonZeros() & size_t(-4) );
928  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
929 
930  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
931  {
932  const size_t j1( element->index() );
933  const VET v1( element->value() );
934  ++element;
935  const size_t j2( element->index() );
936  const VET v2( element->value() );
937  ++element;
938  const size_t j3( element->index() );
939  const VET v3( element->value() );
940  ++element;
941  const size_t j4( element->index() );
942  const VET v4( element->value() );
943  ++element;
944 
945  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
946 
947  const size_t ibegin( ( IsLower<MT1>::value )
948  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
949  :( 0UL ) );
950  const size_t iend( ( IsUpper<MT1>::value )
951  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
952  :( M ) );
953  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
954 
955  for( size_t i=ibegin; i<iend; ++i ) {
956  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
957  }
958  }
959  for( ; element!=end; ++element )
960  {
961  const size_t j1( element->index() );
962  const VET v1( element->value() );
963 
964  const size_t ibegin( ( IsLower<MT1>::value )
965  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
966  :( 0UL ) );
967  const size_t iend( ( IsUpper<MT1>::value )
968  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
969  :( M ) );
970  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
971 
972  for( size_t i=ibegin; i<iend; ++i ) {
973  y[i] += A(i,j1) * v1;
974  }
975  }
976  }
978  //**********************************************************************************************
979 
980  //**Vectorized addition assignment to dense vectors*********************************************
994  template< typename VT1 // Type of the left-hand side target vector
995  , typename MT1 // Type of the left-hand side matrix operand
996  , typename VT2 > // Type of the right-hand side vector operand
997  static inline EnableIf_< UseVectorizedKernel<VT1,MT1,VT2> >
998  selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
999  {
1000  typedef ConstIterator_< RemoveReference_<RT> > ConstIterator;
1001 
1002  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1003 
1004  const size_t M( A.rows() );
1005 
1006  const bool remainder( !IsPadded<MT1>::value || !IsPadded<VT1>::value );
1007 
1008  ConstIterator element( x.begin() );
1009  const ConstIterator end( x.end() );
1010 
1011  const size_t jpos( x.nonZeros() & size_t(-4) );
1012  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1013 
1014  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1015  {
1016  const size_t j1( element->index() );
1017  const VET v1( element->value() );
1018  ++element;
1019  const size_t j2( element->index() );
1020  const VET v2( element->value() );
1021  ++element;
1022  const size_t j3( element->index() );
1023  const VET v3( element->value() );
1024  ++element;
1025  const size_t j4( element->index() );
1026  const VET v4( element->value() );
1027  ++element;
1028 
1029  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1030 
1031  const SIMDType xmm1( set( v1 ) );
1032  const SIMDType xmm2( set( v2 ) );
1033  const SIMDType xmm3( set( v3 ) );
1034  const SIMDType xmm4( set( v4 ) );
1035 
1036  const size_t ibegin( ( IsLower<MT1>::value )
1037  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1038  :( 0UL ) );
1039  const size_t iend( ( IsUpper<MT1>::value )
1040  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
1041  :( M ) );
1042  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1043 
1044  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1045  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1046 
1047  size_t i( ibegin );
1048 
1049  for( ; i<ipos; i+=SIMDSIZE ) {
1050  y.store( i, y.load(i) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
1051  }
1052  for( ; remainder && i<iend; ++i ) {
1053  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1054  }
1055  }
1056  for( ; element!=end; ++element )
1057  {
1058  const size_t j1( element->index() );
1059  const VET v1( element->value() );
1060 
1061  const SIMDType xmm1( set( v1 ) );
1062 
1063  const size_t ibegin( ( IsLower<MT1>::value )
1064  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1065  :( 0UL ) );
1066  const size_t iend( ( IsUpper<MT1>::value )
1067  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
1068  :( M ) );
1069  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1070 
1071  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1072  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1073 
1074  size_t i( ibegin );
1075 
1076  for( ; i<ipos; i+=SIMDSIZE ) {
1077  y.store( i, y.load(i) + A.load(i,j1) * xmm1 );
1078  }
1079  for( ; remainder && i<iend; ++i ) {
1080  y[i] += A(i,j1) * v1;
1081  }
1082  }
1083  }
1085  //**********************************************************************************************
1086 
1087  //**Addition assignment to sparse vectors*******************************************************
1088  // No special implementation for the addition assignment to sparse vectors.
1089  //**********************************************************************************************
1090 
1091  //**Subtraction assignment to dense vectors*****************************************************
1104  template< typename VT1 > // Type of the target dense vector
1105  friend inline void subAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1106  {
1108 
1109  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1110 
1111  // Evaluation of the right-hand side sparse vector operand
1112  RT x( serial( rhs.vec_ ) );
1113  if( x.nonZeros() == 0UL ) return;
1114 
1115  // Evaluation of the left-hand side dense matrix operand
1116  LT A( serial( rhs.mat_ ) );
1117 
1118  // Checking the evaluated operands
1119  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1120  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1121  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1122  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1123 
1124  // Performing the dense matrix-sparse vector multiplication
1125  TDMatSVecMultExpr::selectSubAssignKernel( ~lhs, A, x );
1126  }
1128  //**********************************************************************************************
1129 
1130  //**Default subtraction assignment to dense vectors*********************************************
1144  template< typename VT1 // Type of the left-hand side target vector
1145  , typename MT1 // Type of the left-hand side matrix operand
1146  , typename VT2 > // Type of the right-hand side vector operand
1147  static inline EnableIf_< UseDefaultKernel<VT1,MT1,VT2> >
1148  selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1149  {
1150  typedef ConstIterator_< RemoveReference_<RT> > ConstIterator;
1151 
1152  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1153 
1154  const size_t M( A.rows() );
1155 
1156  ConstIterator element( x.begin() );
1157  const ConstIterator end( x.end() );
1158 
1159  for( ; element!=end; ++element )
1160  {
1161  const size_t index( element->index() );
1162 
1163  if( IsDiagonal<MT1>::value )
1164  {
1165  y[index] -= A(index,index) * element->value();
1166  }
1167  else
1168  {
1169  const size_t ibegin( ( IsLower<MT1>::value )
1170  ?( IsStrictlyLower<MT1>::value ? index+1UL : index )
1171  :( 0UL ) );
1172  const size_t iend( ( IsUpper<MT1>::value )
1173  ?( IsStrictlyUpper<MT1>::value ? index : index+1UL )
1174  :( M ) );
1175  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1176 
1177  for( size_t i=ibegin; i<iend; ++i ) {
1178  y[i] -= A(i,index) * element->value();
1179  }
1180  }
1181  }
1182  }
1184  //**********************************************************************************************
1185 
1186  //**Optimized subtraction assignment to dense vectors*******************************************
1200  template< typename VT1 // Type of the left-hand side target vector
1201  , typename MT1 // Type of the left-hand side matrix operand
1202  , typename VT2 > // Type of the right-hand side vector operand
1203  static inline EnableIf_< UseOptimizedKernel<VT1,MT1,VT2> >
1204  selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1205  {
1206  typedef ConstIterator_< RemoveReference_<RT> > ConstIterator;
1207 
1208  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1209 
1210  const size_t M( A.rows() );
1211 
1212  ConstIterator element( x.begin() );
1213  const ConstIterator end( x.end() );
1214 
1215  const size_t jpos( x.nonZeros() & size_t(-4) );
1216  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1217 
1218  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1219  {
1220  const size_t j1( element->index() );
1221  const VET v1( element->value() );
1222  ++element;
1223  const size_t j2( element->index() );
1224  const VET v2( element->value() );
1225  ++element;
1226  const size_t j3( element->index() );
1227  const VET v3( element->value() );
1228  ++element;
1229  const size_t j4( element->index() );
1230  const VET v4( element->value() );
1231  ++element;
1232 
1233  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1234 
1235  const size_t ibegin( ( IsLower<MT1>::value )
1236  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
1237  :( 0UL ) );
1238  const size_t iend( ( IsUpper<MT1>::value )
1239  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
1240  :( M ) );
1241  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1242 
1243  for( size_t i=ibegin; i<iend; ++i ) {
1244  y[i] -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1245  }
1246  }
1247  for( ; element!=end; ++element )
1248  {
1249  const size_t j1( element->index() );
1250  const VET v1( element->value() );
1251 
1252  const size_t ibegin( ( IsLower<MT1>::value )
1253  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
1254  :( 0UL ) );
1255  const size_t iend( ( IsUpper<MT1>::value )
1256  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
1257  :( M ) );
1258  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1259 
1260  for( size_t i=ibegin; i<iend; ++i ) {
1261  y[i] -= A(i,j1) * v1;
1262  }
1263  }
1264  }
1266  //**********************************************************************************************
1267 
1268  //**Vectorized subtraction assignment to dense vectors******************************************
1282  template< typename VT1 // Type of the left-hand side target vector
1283  , typename MT1 // Type of the left-hand side matrix operand
1284  , typename VT2 > // Type of the right-hand side vector operand
1285  static inline EnableIf_< UseVectorizedKernel<VT1,MT1,VT2> >
1286  selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1287  {
1288  typedef ConstIterator_< RemoveReference_<RT> > ConstIterator;
1289 
1290  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1291 
1292  const size_t M( A.rows() );
1293 
1294  const bool remainder( !IsPadded<MT1>::value || !IsPadded<VT1>::value );
1295 
1296  ConstIterator element( x.begin() );
1297  const ConstIterator end( x.end() );
1298 
1299  const size_t jpos( x.nonZeros() & size_t(-4) );
1300  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1301 
1302  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1303  {
1304  const size_t j1( element->index() );
1305  const VET v1( element->value() );
1306  ++element;
1307  const size_t j2( element->index() );
1308  const VET v2( element->value() );
1309  ++element;
1310  const size_t j3( element->index() );
1311  const VET v3( element->value() );
1312  ++element;
1313  const size_t j4( element->index() );
1314  const VET v4( element->value() );
1315  ++element;
1316 
1317  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1318 
1319  const SIMDType xmm1( set( v1 ) );
1320  const SIMDType xmm2( set( v2 ) );
1321  const SIMDType xmm3( set( v3 ) );
1322  const SIMDType xmm4( set( v4 ) );
1323 
1324  const size_t ibegin( ( IsLower<MT1>::value )
1325  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1326  :( 0UL ) );
1327  const size_t iend( ( IsUpper<MT1>::value )
1328  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
1329  :( M ) );
1330  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1331 
1332  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1333  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1334 
1335  size_t i( ibegin );
1336 
1337  for( ; i<ipos; i+=SIMDSIZE ) {
1338  y.store( i, y.load(i) - A.load(i,j1) * xmm1 - A.load(i,j2) * xmm2 - A.load(i,j3) * xmm3 - A.load(i,j4) * xmm4 );
1339  }
1340  for( ; remainder && i<iend; ++i ) {
1341  y[i] -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1342  }
1343  }
1344  for( ; element!=end; ++element )
1345  {
1346  const size_t j1( element->index() );
1347  const VET v1( element->value() );
1348 
1349  const SIMDType xmm1( set( v1 ) );
1350 
1351  const size_t ibegin( ( IsLower<MT1>::value )
1352  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1353  :( 0UL ) );
1354  const size_t iend( ( IsUpper<MT1>::value )
1355  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
1356  :( M ) );
1357  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1358 
1359  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1360  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1361 
1362  size_t i( ibegin );
1363 
1364  for( ; i<ipos; i+=SIMDSIZE ) {
1365  y.store( i, y.load(i) - A.load(i,j1) * xmm1 );
1366  }
1367  for( ; remainder && i<iend; ++i ) {
1368  y[i] -= A(i,j1) * v1;
1369  }
1370  }
1371  }
1373  //**********************************************************************************************
1374 
1375  //**Subtraction assignment to sparse vectors****************************************************
1376  // No special implementation for the subtraction assignment to sparse vectors.
1377  //**********************************************************************************************
1378 
1379  //**Multiplication assignment to dense vectors**************************************************
1392  template< typename VT1 > // Type of the target dense vector
1393  friend inline void multAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1394  {
1396 
1399  BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE( CompositeType_<ResultType> );
1400 
1401  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1402 
1403  const ResultType tmp( serial( rhs ) );
1404  multAssign( ~lhs, tmp );
1405  }
1407  //**********************************************************************************************
1408 
1409  //**Multiplication assignment to sparse vectors*************************************************
1410  // No special implementation for the multiplication assignment to sparse vectors.
1411  //**********************************************************************************************
1412 
1413  //**Division assignment to dense vectors********************************************************
1426  template< typename VT1 > // Type of the target dense vector
1427  friend inline void divAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1428  {
1430 
1433  BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE( CompositeType_<ResultType> );
1434 
1435  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1436 
1437  const ResultType tmp( serial( rhs ) );
1438  divAssign( ~lhs, tmp );
1439  }
1441  //**********************************************************************************************
1442 
1443  //**Division assignment to sparse vectors*******************************************************
1444  // No special implementation for the division assignment to sparse vectors.
1445  //**********************************************************************************************
1446 
1447  //**SMP assignment to dense vectors*************************************************************
1462  template< typename VT1 > // Type of the target dense vector
1463  friend inline EnableIf_< UseSMPAssign<VT1> >
1464  smpAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1465  {
1467 
1468  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1469 
1470  // Evaluation of the right-hand side sparse vector operand
1471  RT x( rhs.vec_ );
1472  if( x.nonZeros() == 0UL ) {
1473  reset( ~lhs );
1474  return;
1475  }
1476 
1477  // Evaluation of the left-hand side dense matrix operand
1478  LT A( rhs.mat_ );
1479 
1480  // Checking the evaluated operands
1481  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1482  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1483  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1484  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1485 
1486  // Performing the dense matrix-sparse vector multiplication
1487  smpAssign( ~lhs, A * x );
1488  }
1490  //**********************************************************************************************
1491 
1492  //**SMP assignment to sparse vectors************************************************************
1507  template< typename VT1 > // Type of the target sparse vector
1508  friend inline EnableIf_< UseSMPAssign<VT1> >
1509  smpAssign( SparseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1510  {
1512 
1515  BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE( CompositeType_<ResultType> );
1516 
1517  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1518 
1519  const ResultType tmp( rhs );
1520  smpAssign( ~lhs, tmp );
1521  }
1523  //**********************************************************************************************
1524 
1525  //**SMP addition assignment to dense vectors****************************************************
1540  template< typename VT1 > // Type of the target dense vector
1541  friend inline EnableIf_< UseSMPAssign<VT1> >
1542  smpAddAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1543  {
1545 
1546  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1547 
1548  // Evaluation of the right-hand side sparse vector operand
1549  RT x( rhs.vec_ );
1550  if( x.nonZeros() == 0UL ) return;
1551 
1552  // Evaluation of the left-hand side dense matrix operand
1553  LT A( rhs.mat_ );
1554 
1555  // Checking the evaluated operands
1556  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1557  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1558  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1559  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1560 
1561  // Performing the dense matrix-sparse vector multiplication
1562  smpAddAssign( ~lhs, A * x );
1563  }
1565  //**********************************************************************************************
1566 
1567  //**SMP addition assignment to sparse vectors***************************************************
1568  // No special implementation for the SMP addition assignment to sparse vectors.
1569  //**********************************************************************************************
1570 
1571  //**SMP subtraction assignment to dense vectors*************************************************
1586  template< typename VT1 > // Type of the target dense vector
1587  friend inline EnableIf_< UseSMPAssign<VT1> >
1588  smpSubAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1589  {
1591 
1592  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1593 
1594  // Evaluation of the right-hand side sparse vector operand
1595  RT x( rhs.vec_ );
1596  if( x.nonZeros() == 0UL ) return;
1597 
1598  // Evaluation of the left-hand side dense matrix operand
1599  LT A( rhs.mat_ );
1600 
1601  // Checking the evaluated operands
1602  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1603  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1604  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1605  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1606 
1607  // Performing the dense matrix-sparse vector multiplication
1608  smpSubAssign( ~lhs, A * x );
1609  }
1611  //**********************************************************************************************
1612 
1613  //**SMP subtraction assignment to sparse vectors************************************************
1614  // No special implementation for the SMP subtraction assignment to sparse vectors.
1615  //**********************************************************************************************
1616 
1617  //**SMP multiplication assignment to dense vectors**********************************************
1632  template< typename VT1 > // Type of the target dense vector
1633  friend inline EnableIf_< UseSMPAssign<VT1> >
1634  smpMultAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1635  {
1637 
1640  BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE( CompositeType_<ResultType> );
1641 
1642  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1643 
1644  const ResultType tmp( rhs );
1645  smpMultAssign( ~lhs, tmp );
1646  }
1648  //**********************************************************************************************
1649 
1650  //**SMP multiplication assignment to sparse vectors*********************************************
1651  // No special implementation for the SMP multiplication assignment to sparse vectors.
1652  //**********************************************************************************************
1653 
1654  //**SMP division assignment to dense vectors****************************************************
1669  template< typename VT1 > // Type of the target dense vector
1670  friend inline EnableIf_< UseSMPAssign<VT1> >
1671  smpDivAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1672  {
1674 
1677  BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE( CompositeType_<ResultType> );
1678 
1679  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1680 
1681  const ResultType tmp( rhs );
1682  smpDivAssign( ~lhs, tmp );
1683  }
1685  //**********************************************************************************************
1686 
1687  //**SMP division assignment to sparse vectors***************************************************
1688  // No special implementation for the SMP division assignment to sparse vectors.
1689  //**********************************************************************************************
1690 
1691  //**Compile time checks*************************************************************************
1699  //**********************************************************************************************
1700 };
1701 //*************************************************************************************************
1702 
1703 
1704 
1705 
1706 //=================================================================================================
1707 //
1708 // GLOBAL BINARY ARITHMETIC OPERATORS
1709 //
1710 //=================================================================================================
1711 
1712 //*************************************************************************************************
1744 template< typename T1 // Type of the left-hand side dense matrix
1745  , typename T2 > // Type of the right-hand side sparse vector
1746 inline const DisableIf_< IsMatMatMultExpr<T1>, TDMatSVecMultExpr<T1,T2> >
1748 {
1750 
1751  if( (~mat).columns() != (~vec).size() ) {
1752  BLAZE_THROW_INVALID_ARGUMENT( "Matrix and vector sizes do not match" );
1753  }
1754 
1755  return TDMatSVecMultExpr<T1,T2>( ~mat, ~vec );
1756 }
1757 //*************************************************************************************************
1758 
1759 
1760 
1761 
1762 //=================================================================================================
1763 //
1764 // SIZE SPECIALIZATIONS
1765 //
1766 //=================================================================================================
1767 
1768 //*************************************************************************************************
1770 template< typename MT, typename VT >
1771 struct Size< TDMatSVecMultExpr<MT,VT> > : public Rows<MT>
1772 {};
1774 //*************************************************************************************************
1775 
1776 
1777 
1778 
1779 //=================================================================================================
1780 //
1781 // ISALIGNED SPECIALIZATIONS
1782 //
1783 //=================================================================================================
1784 
1785 //*************************************************************************************************
1787 template< typename MT, typename VT >
1788 struct IsAligned< TDMatSVecMultExpr<MT,VT> >
1789  : public BoolConstant< IsAligned<MT>::value >
1790 {};
1792 //*************************************************************************************************
1793 
1794 
1795 
1796 
1797 //=================================================================================================
1798 //
1799 // EXPRESSION TRAIT SPECIALIZATIONS
1800 //
1801 //=================================================================================================
1802 
1803 //*************************************************************************************************
1805 template< typename MT, typename VT, bool AF >
1806 struct SubvectorExprTrait< TDMatSVecMultExpr<MT,VT>, AF >
1807 {
1808  public:
1809  //**********************************************************************************************
1810  using Type = MultExprTrait_< SubmatrixExprTrait_<const MT,AF>
1811  , SubvectorExprTrait_<const VT,AF> >;
1812  //**********************************************************************************************
1813 };
1815 //*************************************************************************************************
1816 
1817 } // namespace blaze
1818 
1819 #endif
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Header file for auxiliary alias declarations.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:72
RightOperand vec_
Right-hand side sparse vector of the multiplication expression.
Definition: TDMatSVecMultExpr.h:373
constexpr bool useOptimizedKernels
Configuration switch for optimized kernels.This configuration switch enables/disables all optimized c...
Definition: Optimizations.h:84
CompositeType_< VT > VCT
Composite type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:121
Header file for the Rows type trait.
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:7800
Header file for basic type definitions.
CompositeType_< MT > MCT
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:120
ElementType_< ResultType > ElementType
Resulting element type.
Definition: TDMatSVecMultExpr.h:201
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a column dense or sparse vector type...
Definition: ColumnVector.h:61
EnableIf_< IsDenseMatrix< MT1 > > smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
IfTrue_< evaluateVector, const VRT, CompositeType_< VT > > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: TDMatSVecMultExpr.h:216
Header file for the serial shim.
Header file for the IsDiagonal type trait.
BLAZE_ALWAYS_INLINE size_t size(const Vector< VT, TF > &vector) noexcept
Returns the current size/dimension of the vector.
Definition: Vector.h:258
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the IsSame and IsStrictlySame type traits.
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:188
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:162
MultTrait_< MRT, VRT > ResultType
Result type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:199
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:315
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:533
EnableIf_< IsDenseVector< VT1 > > smpMultAssign(Vector< VT1, TF1 > &lhs, const Vector< VT2, TF2 > &rhs)
Default implementation of the SMP multiplication assignment of a vector to a dense vector...
Definition: DenseVector.h:193
ElementType_< VRT > VET
Element type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:119
Header file for the DenseVector base class.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:88
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:162
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:723
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:245
Header file for the Computation base class.
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:203
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:88
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatSVecMultExpr.h:355
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:323
EnableIf_< IsDenseMatrix< MT1 > > smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
Constraint on the data type.
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
IfTrue_< evaluateMatrix, const MRT, MCT > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatSVecMultExpr.h:213
Constraint on the transpose flag of vector types.
Constraint on the data type.
Header file for the MultExprTrait class template.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:72
SubvectorExprTrait_< VT, unaligned > subvector(Vector< VT, TF > &vector, size_t index, size_t size)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:152
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TDMatSVecMultExpr.h:345
Header file for the DisableIf class template.
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
ResultType_< MT > MRT
Result type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:116
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
Header file for the If class template.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2647
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Header file for the IsMatMatMultExpr type trait class.
LeftOperand mat_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatSVecMultExpr.h:372
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:200
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
If_< IsExpression< MT >, const MT, const MT & > LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:207
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
LeftOperand leftOperand() const noexcept
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatSVecMultExpr.h:311
Base class for N-dimensional dense vectors.The DenseVector class is a base class for all arbitrarily ...
Definition: DenseVector.h:70
Header file for the IsLower type trait.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional vector type...
Definition: SparseVector.h:61
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:90
BLAZE_ALWAYS_INLINE size_t columns(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of columns of the matrix.
Definition: Matrix.h:330
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:60
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATVECMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/vector ...
Definition: MatVecMultExpr.h:110
Constraint on the data type.
Header file for the exception macros of the math module.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:254
Header file for all forward declarations for expression class templates.
Constraint on the data type.
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
If_< IsExpression< VT >, const VT, const VT & > RightOperand
Composite type of the right-hand side dense vector expression.
Definition: TDMatSVecMultExpr.h:210
Expression object for transpose dense matrix-sparse vector multiplications.The TDMatSVecMultExpr clas...
Definition: Forward.h:133
BLAZE_ALWAYS_INLINE const EnableIf_< And< IsIntegral< T >, HasSize< T, 1UL > >, If_< IsSigned< T >, SIMDint8, SIMDuint8 > > set(T value) noexcept
Sets all values in the vector to the given 1-byte integral value.
Definition: Set.h:76
DisableIf_< Or< IsComputation< MT >, IsTransExpr< MT > >, RowExprTrait_< MT > > row(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific row of the given matrix.
Definition: Row.h:126
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: TDMatSVecMultExpr.h:202
Header file for the SubmatrixExprTrait class template.
Header file for the HasSIMDMult type trait.
Header file for run time assertion macros.
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:160
EnableIf_< IsDenseVector< VT1 > > smpDivAssign(Vector< VT1, TF1 > &lhs, const Vector< VT2, TF2 > &rhs)
Default implementation of the SMP division assignment of a vector to a dense vector.
Definition: DenseVector.h:222
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatSVecMultExpr.h:204
Header file for the reset shim.
RightOperand rightOperand() const noexcept
Returns the right-hand side sparse vector operand.
Definition: TDMatSVecMultExpr.h:321
Constraints on the storage order of matrix types.
ResultType_< VT > VRT
Result type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:117
TDMatSVecMultExpr(const MT &mat, const VT &vec) noexcept
Constructor for the TDMatSVecMultExpr class.
Definition: TDMatSVecMultExpr.h:242
IntegralConstant< bool, B > BoolConstant
Generic wrapper for a compile time constant boolean value.The BoolConstant class template represents ...
Definition: IntegralConstant.h:100
Header file for the RemoveReference type trait.
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:223
ElementType_< MRT > MET
Element type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:118
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional vector type...
Definition: DenseVector.h:61
Header file for the AreSIMDCombinable type trait.
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TDMatSVecMultExpr.h:333
ReturnType at(size_t index) const
Checked access to the vector elements.
Definition: TDMatSVecMultExpr.h:288
Header file for the IsComputation type trait class.
TDMatSVecMultExpr< MT, VT > This
Type of this TDMatSVecMultExpr instance.
Definition: TDMatSVecMultExpr.h:198
Base class for sparse vectors.The SparseVector class is a base class for all arbitrarily sized (N-dim...
Definition: Forward.h:110
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
Header file for the IntegralConstant class template.
Header file for the SubvectorExprTrait class template.
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TDMatSVecMultExpr.h:365
Constraint on the data type.
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:403
Header file for the IsUpper type trait.
Header file for the MatVecMultExpr base class.
ReturnType operator[](size_t index) const
Subscript operator for the direct access to the vector elements.
Definition: TDMatSVecMultExpr.h:256
Header file for the IsResizable type trait.
Header file for the Size type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the IsExpression type trait class.
Header file for the FunctionTrace class.
size_t size() const noexcept
Returns the current size/dimension of the vector.
Definition: TDMatSVecMultExpr.h:301