TDMatSVecMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATSVECMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATSVECMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
53 #include <blaze/math/Exception.h>
58 #include <blaze/math/shims/Reset.h>
60 #include <blaze/math/SIMD.h>
79 #include <blaze/util/Assert.h>
80 #include <blaze/util/DisableIf.h>
81 #include <blaze/util/EnableIf.h>
84 #include <blaze/util/mpl/If.h>
85 #include <blaze/util/Types.h>
88 
89 
90 namespace blaze {
91 
92 //=================================================================================================
93 //
94 // CLASS TDMATSVECMULTEXPR
95 //
96 //=================================================================================================
97 
98 //*************************************************************************************************
105 template< typename MT // Type of the left-hand side dense matrix
106  , typename VT > // Type of the right-hand side sparse vector
107 class TDMatSVecMultExpr
108  : public MatVecMultExpr< DenseVector< TDMatSVecMultExpr<MT,VT>, false > >
109  , private Computation
110 {
111  private:
112  //**Type definitions****************************************************************************
119  //**********************************************************************************************
120 
121  //**********************************************************************************************
123  enum : bool { evaluateMatrix = RequiresEvaluation<MT>::value };
124  //**********************************************************************************************
125 
126  //**********************************************************************************************
128  enum : bool { evaluateVector = IsComputation<VT>::value || RequiresEvaluation<VT>::value };
129  //**********************************************************************************************
130 
131  //**********************************************************************************************
133 
137  template< typename T1 >
138  struct UseSMPAssign {
139  enum : bool { value = ( evaluateMatrix || evaluateVector ) };
140  };
142  //**********************************************************************************************
143 
144  //**********************************************************************************************
146 
150  template< typename T1, typename T2, typename T3 >
151  struct UseVectorizedKernel {
152  enum : bool { value = useOptimizedKernels &&
154  T1::simdEnabled && T2::simdEnabled &&
157  , ElementType_<T3> >::value &&
160  };
162  //**********************************************************************************************
163 
164  //**********************************************************************************************
166 
170  template< typename T1, typename T2, typename T3 >
171  struct UseOptimizedKernel {
172  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
174  !IsResizable< ElementType_<T1> >::value &&
176  };
178  //**********************************************************************************************
179 
180  //**********************************************************************************************
182 
185  template< typename T1, typename T2, typename T3 >
186  struct UseDefaultKernel {
187  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
188  !UseOptimizedKernel<T1,T2,T3>::value };
189  };
191  //**********************************************************************************************
192 
193  public:
194  //**Type definitions****************************************************************************
200  using ReturnType = const ElementType;
201  using CompositeType = const ResultType;
202 
204  using LeftOperand = If_< IsExpression<MT>, const MT, const MT& >;
205 
207  using RightOperand = If_< IsExpression<VT>, const VT, const VT& >;
208 
211 
214  //**********************************************************************************************
215 
216  //**Compilation flags***************************************************************************
218  enum : bool { simdEnabled = !IsDiagonal<MT>::value &&
219  MT::simdEnabled &&
222 
224  enum : bool { smpAssignable = !evaluateMatrix && MT::smpAssignable &&
225  !evaluateVector && VT::smpAssignable };
226  //**********************************************************************************************
227 
228  //**SIMD properties*****************************************************************************
230  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
231  //**********************************************************************************************
232 
233  //**Constructor*********************************************************************************
239  explicit inline TDMatSVecMultExpr( const MT& mat, const VT& vec ) noexcept
240  : mat_( mat ) // Left-hand side dense matrix of the multiplication expression
241  , vec_( vec ) // Right-hand side sparse vector of the multiplication expression
242  {
243  BLAZE_INTERNAL_ASSERT( mat_.columns() == vec_.size(), "Invalid matrix and vector sizes" );
244  }
245  //**********************************************************************************************
246 
247  //**Subscript operator**************************************************************************
253  inline ReturnType operator[]( size_t index ) const {
254  BLAZE_INTERNAL_ASSERT( index < mat_.rows(), "Invalid vector access index" );
255 
257  {
258  return mat_(index,index) * vec_[index];
259  }
260  else if( IsLower<MT>::value )
261  {
262  const size_t n( IsStrictlyLower<MT>::value ? index : index+1UL );
263  return subvector( row( mat_, index ), 0UL, n ) * subvector( vec_, 0UL, n );
264  }
265  else if( IsUpper<MT>::value )
266  {
267  const size_t begin( IsStrictlyUpper<MT>::value ? index+1UL : index );
268  const size_t n ( mat_.columns() - begin );
269  return subvector( row( mat_, index ), begin, n ) * subvector( vec_, begin, n );
270  }
271  else
272  {
273  return row( mat_, index ) * vec_;
274  }
275  }
276  //**********************************************************************************************
277 
278  //**At function*********************************************************************************
285  inline ReturnType at( size_t index ) const {
286  if( index >= mat_.rows() ) {
287  BLAZE_THROW_OUT_OF_RANGE( "Invalid vector access index" );
288  }
289  return (*this)[index];
290  }
291  //**********************************************************************************************
292 
293  //**Size function*******************************************************************************
298  inline size_t size() const noexcept {
299  return mat_.rows();
300  }
301  //**********************************************************************************************
302 
303  //**Left operand access*************************************************************************
308  inline LeftOperand leftOperand() const noexcept {
309  return mat_;
310  }
311  //**********************************************************************************************
312 
313  //**Right operand access************************************************************************
318  inline RightOperand rightOperand() const noexcept {
319  return vec_;
320  }
321  //**********************************************************************************************
322 
323  //**********************************************************************************************
329  template< typename T >
330  inline bool canAlias( const T* alias ) const noexcept {
331  return mat_.isAliased( alias ) || vec_.isAliased( alias );
332  }
333  //**********************************************************************************************
334 
335  //**********************************************************************************************
341  template< typename T >
342  inline bool isAliased( const T* alias ) const noexcept {
343  return mat_.isAliased( alias ) || vec_.isAliased( alias );
344  }
345  //**********************************************************************************************
346 
347  //**********************************************************************************************
352  inline bool isAligned() const noexcept {
353  return mat_.isAligned();
354  }
355  //**********************************************************************************************
356 
357  //**********************************************************************************************
362  inline bool canSMPAssign() const noexcept {
363  return ( size() > SMP_TDMATSVECMULT_THRESHOLD );
364  }
365  //**********************************************************************************************
366 
367  private:
368  //**Member variables****************************************************************************
371  //**********************************************************************************************
372 
373  //**Assignment to dense vectors*****************************************************************
386  template< typename VT1 > // Type of the target dense vector
387  friend inline void assign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
388  {
390 
391  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
392 
393  // Evaluation of the right-hand side sparse vector operand
394  RT x( serial( rhs.vec_ ) );
395  if( x.nonZeros() == 0UL ) {
396  reset( ~lhs );
397  return;
398  }
399 
400  // Evaluation of the left-hand side dense matrix operand
401  LT A( serial( rhs.mat_ ) );
402 
403  // Checking the evaluated operands
404  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
405  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
406  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
407  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
408 
409  // Performing the dense matrix-sparse vector multiplication
410  TDMatSVecMultExpr::selectAssignKernel( ~lhs, A, x );
411  }
413  //**********************************************************************************************
414 
415  //**Default assignment to dense vectors*********************************************************
429  template< typename VT1 // Type of the left-hand side target vector
430  , typename MT1 // Type of the left-hand side matrix operand
431  , typename VT2 > // Type of the right-hand side vector operand
433  selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
434  {
436 
437  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
438 
439  const size_t M( A.rows() );
440 
441  ConstIterator element( x.begin() );
442  const ConstIterator end( x.end() );
443 
444  size_t last( 0UL );
445 
446  if( IsLower<MT1>::value ) {
447  const size_t iend( IsStrictlyLower<MT1>::value ? element->index()+1UL : element->index() );
448  for( size_t i=0UL; i<iend; ++i )
449  reset( y[i] );
450  }
451 
452  for( ; element!=end; ++element )
453  {
454  const size_t index( element->index() );
455 
457  {
458  for( size_t i=last; i<index; ++i )
459  reset( y[i] );
460 
461  y[index] = A(index,index) * element->value();
462  last = index + 1UL;
463  }
464  else
465  {
466  const size_t ibegin( ( IsLower<MT1>::value )
467  ?( IsStrictlyLower<MT1>::value ? index+1UL : index )
468  :( 0UL ) );
469  const size_t iend( ( IsUpper<MT1>::value )
470  ?( IsStrictlyUpper<MT1>::value ? index : index+1UL )
471  :( M ) );
472  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
473 
474  for( size_t i=ibegin; i<last; ++i ) {
475  y[i] += A(i,index) * element->value();
476  }
477  for( size_t i=last; i<iend; ++i ) {
478  y[i] = A(i,index) * element->value();
479  }
480 
481  last = iend;
482  }
483  }
484 
485  if( IsUpper<MT1>::value ) {
486  for( size_t i=last; i<M; ++i )
487  reset( y[i] );
488  }
489  }
491  //**********************************************************************************************
492 
493  //**Optimized assignment to dense vectors*******************************************************
507  template< typename VT1 // Type of the left-hand side target vector
508  , typename MT1 // Type of the left-hand side matrix operand
509  , typename VT2 > // Type of the right-hand side vector operand
511  selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
512  {
514 
515  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
516 
517  const size_t M( A.rows() );
518 
519  ConstIterator element( x.begin() );
520  const ConstIterator end( x.end() );
521 
522  const size_t jpos( x.nonZeros() & size_t(-4) );
523  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
524 
525  if( jpos > 3UL )
526  {
527  const size_t j1( element->index() );
528  const VET v1( element->value() );
529  ++element;
530  const size_t j2( element->index() );
531  const VET v2( element->value() );
532  ++element;
533  const size_t j3( element->index() );
534  const VET v3( element->value() );
535  ++element;
536  const size_t j4( element->index() );
537  const VET v4( element->value() );
538  ++element;
539 
540  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
541 
542  for( size_t i=0UL; i<M; ++i ) {
543  y[i] = A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
544  }
545  }
546  else
547  {
548  const size_t j1( element->index() );
549  const VET v1( element->value() );
550  ++element;
551 
552  for( size_t i=0UL; i<M; ++i ) {
553  y[i] = A(i,j1) * v1;
554  }
555  }
556 
557  for( size_t j=(jpos>3UL)?(4UL):(1UL); (j+4UL)<=jpos; j+=4UL )
558  {
559  const size_t j1( element->index() );
560  const VET v1( element->value() );
561  ++element;
562  const size_t j2( element->index() );
563  const VET v2( element->value() );
564  ++element;
565  const size_t j3( element->index() );
566  const VET v3( element->value() );
567  ++element;
568  const size_t j4( element->index() );
569  const VET v4( element->value() );
570  ++element;
571 
572  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
573 
574  const size_t ibegin( ( IsLower<MT1>::value )
575  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
576  :( 0UL ) );
577  const size_t iend( ( IsUpper<MT1>::value )
578  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
579  :( M ) );
580  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
581 
582  for( size_t i=ibegin; i<iend; ++i ) {
583  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
584  }
585  }
586  for( ; element!=end; ++element )
587  {
588  const size_t j1( element->index() );
589  const VET v1( element->value() );
590 
591  const size_t ibegin( ( IsLower<MT1>::value )
592  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
593  :( 0UL ) );
594  const size_t iend( ( IsUpper<MT1>::value )
595  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
596  :( M ) );
597  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
598 
599  for( size_t i=ibegin; i<iend; ++i ) {
600  y[i] += A(i,j1) * v1;
601  }
602  }
603  }
605  //**********************************************************************************************
606 
607  //**Vectorized assignment to dense vectors******************************************************
621  template< typename VT1 // Type of the left-hand side target vector
622  , typename MT1 // Type of the left-hand side matrix operand
623  , typename VT2 > // Type of the right-hand side vector operand
625  selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
626  {
628 
629  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
630 
631  constexpr bool remainder( !IsPadded<MT1>::value || !IsPadded<VT1>::value );
632 
633  const size_t M( A.rows() );
634 
635  ConstIterator element( x.begin() );
636  const ConstIterator end( x.end() );
637 
638  const size_t jpos( x.nonZeros() & size_t(-4) );
639  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
640 
641  if( jpos > 3UL )
642  {
643  const size_t j1( element->index() );
644  const VET v1( element->value() );
645  ++element;
646  const size_t j2( element->index() );
647  const VET v2( element->value() );
648  ++element;
649  const size_t j3( element->index() );
650  const VET v3( element->value() );
651  ++element;
652  const size_t j4( element->index() );
653  const VET v4( element->value() );
654  ++element;
655 
656  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
657 
658  const SIMDType xmm1( set( v1 ) );
659  const SIMDType xmm2( set( v2 ) );
660  const SIMDType xmm3( set( v3 ) );
661  const SIMDType xmm4( set( v4 ) );
662 
663  const size_t ipos( remainder ? ( M & size_t(-SIMDSIZE) ) : M );
664  BLAZE_INTERNAL_ASSERT( !remainder || ( M - ( M % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
665 
666  size_t i( 0UL );
667 
668  for( ; i<ipos; i+=SIMDSIZE ) {
669  y.store( i, A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
670  }
671  for( ; remainder && i<M; ++i ) {
672  y[i] = A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
673  }
674  }
675  else
676  {
677  const size_t j1( element->index() );
678  const VET v1( element->value() );
679  ++element;
680 
681  const SIMDType xmm1( set( v1 ) );
682 
683  const size_t ipos( remainder ? ( M & size_t(-SIMDSIZE) ) : M );
684  BLAZE_INTERNAL_ASSERT( !remainder || ( M - ( M % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
685 
686  size_t i( 0UL );
687 
688  for( ; i<ipos; i+=SIMDSIZE ) {
689  y.store( i, A.load(i,j1) * xmm1 );
690  }
691  for( ; remainder && i<M; ++i ) {
692  y[i] = A(i,j1) * v1;
693  }
694  }
695 
696  for( size_t j=(jpos>3UL)?(4UL):(1UL); (j+4UL)<=jpos; j+=4UL )
697  {
698  const size_t j1( element->index() );
699  const VET v1( element->value() );
700  ++element;
701  const size_t j2( element->index() );
702  const VET v2( element->value() );
703  ++element;
704  const size_t j3( element->index() );
705  const VET v3( element->value() );
706  ++element;
707  const size_t j4( element->index() );
708  const VET v4( element->value() );
709  ++element;
710 
711  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
712 
713  const SIMDType xmm1( set( v1 ) );
714  const SIMDType xmm2( set( v2 ) );
715  const SIMDType xmm3( set( v3 ) );
716  const SIMDType xmm4( set( v4 ) );
717 
718  const size_t ibegin( ( IsLower<MT1>::value )
719  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
720  :( 0UL ) );
721  const size_t iend( ( IsUpper<MT1>::value )
722  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
723  :( M ) );
724  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
725 
726  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
727  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
728 
729  size_t i( ibegin );
730 
731  for( ; i<ipos; i+=SIMDSIZE ) {
732  y.store( i, y.load(i) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
733  }
734  for( ; remainder && i<iend; ++i ) {
735  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
736  }
737  }
738 
739  for( ; element!=end; ++element )
740  {
741  const size_t j1( element->index() );
742  const VET v1( element->value() );
743 
744  const SIMDType xmm1( set( v1 ) );
745 
746  const size_t ibegin( ( IsLower<MT1>::value )
747  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
748  :( 0UL ) );
749  const size_t iend( ( IsUpper<MT1>::value )
750  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
751  :( M ) );
752  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
753 
754  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
755  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
756 
757  size_t i( ibegin );
758 
759  for( ; i<ipos; i+=SIMDSIZE ) {
760  y.store( i, y.load(i) + A.load(i,j1) * xmm1 );
761  }
762  for( ; remainder && i<iend; ++i ) {
763  y[i] += A(i,j1) * v1;
764  }
765  }
766  }
768  //**********************************************************************************************
769 
770  //**Assignment to sparse vectors****************************************************************
783  template< typename VT1 > // Type of the target sparse vector
784  friend inline void assign( SparseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
785  {
787 
791 
792  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
793 
794  const ResultType tmp( serial( rhs ) );
795  assign( ~lhs, tmp );
796  }
798  //**********************************************************************************************
799 
800  //**Addition assignment to dense vectors********************************************************
813  template< typename VT1 > // Type of the target dense vector
814  friend inline void addAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
815  {
817 
818  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
819 
820  // Evaluation of the right-hand side sparse vector operand
821  RT x( serial( rhs.vec_ ) );
822  if( x.nonZeros() == 0UL ) return;
823 
824  // Evaluation of the left-hand side dense matrix operand
825  LT A( serial( rhs.mat_ ) );
826 
827  // Checking the evaluated operands
828  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
829  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
830  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
831  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
832 
833  // Performing the dense matrix-sparse vector multiplication
834  TDMatSVecMultExpr::selectAddAssignKernel( ~lhs, A, x );
835  }
837  //**********************************************************************************************
838 
839  //**Default addition assignment to dense vectors************************************************
853  template< typename VT1 // Type of the left-hand side target vector
854  , typename MT1 // Type of the left-hand side matrix operand
855  , typename VT2 > // Type of the right-hand side vector operand
857  selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
858  {
860 
861  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
862 
863  const size_t M( A.rows() );
864 
865  ConstIterator element( x.begin() );
866  const ConstIterator end( x.end() );
867 
868  for( ; element!=end; ++element )
869  {
870  const size_t index( element->index() );
871 
873  {
874  y[index] += A(index,index) * element->value();
875  }
876  else
877  {
878  const size_t ibegin( ( IsLower<MT1>::value )
879  ?( IsStrictlyLower<MT1>::value ? index+1UL : index )
880  :( 0UL ) );
881  const size_t iend( ( IsUpper<MT1>::value )
882  ?( IsStrictlyUpper<MT1>::value ? index : index+1UL )
883  :( M ) );
884  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
885 
886  for( size_t i=ibegin; i<iend; ++i ) {
887  y[i] += A(i,index) * element->value();
888  }
889  }
890  }
891  }
893  //**********************************************************************************************
894 
895  //**Optimized addition assignment to dense vectors**********************************************
909  template< typename VT1 // Type of the left-hand side target vector
910  , typename MT1 // Type of the left-hand side matrix operand
911  , typename VT2 > // Type of the right-hand side vector operand
913  selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
914  {
916 
917  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
918 
919  const size_t M( A.rows() );
920 
921  ConstIterator element( x.begin() );
922  const ConstIterator end( x.end() );
923 
924  const size_t jpos( x.nonZeros() & size_t(-4) );
925  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
926 
927  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
928  {
929  const size_t j1( element->index() );
930  const VET v1( element->value() );
931  ++element;
932  const size_t j2( element->index() );
933  const VET v2( element->value() );
934  ++element;
935  const size_t j3( element->index() );
936  const VET v3( element->value() );
937  ++element;
938  const size_t j4( element->index() );
939  const VET v4( element->value() );
940  ++element;
941 
942  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
943 
944  const size_t ibegin( ( IsLower<MT1>::value )
945  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
946  :( 0UL ) );
947  const size_t iend( ( IsUpper<MT1>::value )
948  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
949  :( M ) );
950  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
951 
952  for( size_t i=ibegin; i<iend; ++i ) {
953  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
954  }
955  }
956  for( ; element!=end; ++element )
957  {
958  const size_t j1( element->index() );
959  const VET v1( element->value() );
960 
961  const size_t ibegin( ( IsLower<MT1>::value )
962  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
963  :( 0UL ) );
964  const size_t iend( ( IsUpper<MT1>::value )
965  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
966  :( M ) );
967  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
968 
969  for( size_t i=ibegin; i<iend; ++i ) {
970  y[i] += A(i,j1) * v1;
971  }
972  }
973  }
975  //**********************************************************************************************
976 
977  //**Vectorized addition assignment to dense vectors*********************************************
991  template< typename VT1 // Type of the left-hand side target vector
992  , typename MT1 // Type of the left-hand side matrix operand
993  , typename VT2 > // Type of the right-hand side vector operand
995  selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
996  {
998 
999  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1000 
1001  constexpr bool remainder( !IsPadded<MT1>::value || !IsPadded<VT1>::value );
1002 
1003  const size_t M( A.rows() );
1004 
1005  ConstIterator element( x.begin() );
1006  const ConstIterator end( x.end() );
1007 
1008  const size_t jpos( x.nonZeros() & size_t(-4) );
1009  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1010 
1011  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1012  {
1013  const size_t j1( element->index() );
1014  const VET v1( element->value() );
1015  ++element;
1016  const size_t j2( element->index() );
1017  const VET v2( element->value() );
1018  ++element;
1019  const size_t j3( element->index() );
1020  const VET v3( element->value() );
1021  ++element;
1022  const size_t j4( element->index() );
1023  const VET v4( element->value() );
1024  ++element;
1025 
1026  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1027 
1028  const SIMDType xmm1( set( v1 ) );
1029  const SIMDType xmm2( set( v2 ) );
1030  const SIMDType xmm3( set( v3 ) );
1031  const SIMDType xmm4( set( v4 ) );
1032 
1033  const size_t ibegin( ( IsLower<MT1>::value )
1034  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1035  :( 0UL ) );
1036  const size_t iend( ( IsUpper<MT1>::value )
1037  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
1038  :( M ) );
1039  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1040 
1041  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1042  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1043 
1044  size_t i( ibegin );
1045 
1046  for( ; i<ipos; i+=SIMDSIZE ) {
1047  y.store( i, y.load(i) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
1048  }
1049  for( ; remainder && i<iend; ++i ) {
1050  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1051  }
1052  }
1053  for( ; element!=end; ++element )
1054  {
1055  const size_t j1( element->index() );
1056  const VET v1( element->value() );
1057 
1058  const SIMDType xmm1( set( v1 ) );
1059 
1060  const size_t ibegin( ( IsLower<MT1>::value )
1061  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1062  :( 0UL ) );
1063  const size_t iend( ( IsUpper<MT1>::value )
1064  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
1065  :( M ) );
1066  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1067 
1068  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1069  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1070 
1071  size_t i( ibegin );
1072 
1073  for( ; i<ipos; i+=SIMDSIZE ) {
1074  y.store( i, y.load(i) + A.load(i,j1) * xmm1 );
1075  }
1076  for( ; remainder && i<iend; ++i ) {
1077  y[i] += A(i,j1) * v1;
1078  }
1079  }
1080  }
1082  //**********************************************************************************************
1083 
1084  //**Addition assignment to sparse vectors*******************************************************
1085  // No special implementation for the addition assignment to sparse vectors.
1086  //**********************************************************************************************
1087 
1088  //**Subtraction assignment to dense vectors*****************************************************
1101  template< typename VT1 > // Type of the target dense vector
1102  friend inline void subAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1103  {
1105 
1106  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1107 
1108  // Evaluation of the right-hand side sparse vector operand
1109  RT x( serial( rhs.vec_ ) );
1110  if( x.nonZeros() == 0UL ) return;
1111 
1112  // Evaluation of the left-hand side dense matrix operand
1113  LT A( serial( rhs.mat_ ) );
1114 
1115  // Checking the evaluated operands
1116  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1117  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1118  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1119  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1120 
1121  // Performing the dense matrix-sparse vector multiplication
1122  TDMatSVecMultExpr::selectSubAssignKernel( ~lhs, A, x );
1123  }
1125  //**********************************************************************************************
1126 
1127  //**Default subtraction assignment to dense vectors*********************************************
1141  template< typename VT1 // Type of the left-hand side target vector
1142  , typename MT1 // Type of the left-hand side matrix operand
1143  , typename VT2 > // Type of the right-hand side vector operand
1145  selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1146  {
1148 
1149  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1150 
1151  const size_t M( A.rows() );
1152 
1153  ConstIterator element( x.begin() );
1154  const ConstIterator end( x.end() );
1155 
1156  for( ; element!=end; ++element )
1157  {
1158  const size_t index( element->index() );
1159 
1161  {
1162  y[index] -= A(index,index) * element->value();
1163  }
1164  else
1165  {
1166  const size_t ibegin( ( IsLower<MT1>::value )
1167  ?( IsStrictlyLower<MT1>::value ? index+1UL : index )
1168  :( 0UL ) );
1169  const size_t iend( ( IsUpper<MT1>::value )
1170  ?( IsStrictlyUpper<MT1>::value ? index : index+1UL )
1171  :( M ) );
1172  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1173 
1174  for( size_t i=ibegin; i<iend; ++i ) {
1175  y[i] -= A(i,index) * element->value();
1176  }
1177  }
1178  }
1179  }
1181  //**********************************************************************************************
1182 
1183  //**Optimized subtraction assignment to dense vectors*******************************************
1197  template< typename VT1 // Type of the left-hand side target vector
1198  , typename MT1 // Type of the left-hand side matrix operand
1199  , typename VT2 > // Type of the right-hand side vector operand
1201  selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1202  {
1204 
1205  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1206 
1207  const size_t M( A.rows() );
1208 
1209  ConstIterator element( x.begin() );
1210  const ConstIterator end( x.end() );
1211 
1212  const size_t jpos( x.nonZeros() & size_t(-4) );
1213  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1214 
1215  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1216  {
1217  const size_t j1( element->index() );
1218  const VET v1( element->value() );
1219  ++element;
1220  const size_t j2( element->index() );
1221  const VET v2( element->value() );
1222  ++element;
1223  const size_t j3( element->index() );
1224  const VET v3( element->value() );
1225  ++element;
1226  const size_t j4( element->index() );
1227  const VET v4( element->value() );
1228  ++element;
1229 
1230  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1231 
1232  const size_t ibegin( ( IsLower<MT1>::value )
1233  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
1234  :( 0UL ) );
1235  const size_t iend( ( IsUpper<MT1>::value )
1236  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
1237  :( M ) );
1238  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1239 
1240  for( size_t i=ibegin; i<iend; ++i ) {
1241  y[i] -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1242  }
1243  }
1244  for( ; element!=end; ++element )
1245  {
1246  const size_t j1( element->index() );
1247  const VET v1( element->value() );
1248 
1249  const size_t ibegin( ( IsLower<MT1>::value )
1250  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
1251  :( 0UL ) );
1252  const size_t iend( ( IsUpper<MT1>::value )
1253  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
1254  :( M ) );
1255  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1256 
1257  for( size_t i=ibegin; i<iend; ++i ) {
1258  y[i] -= A(i,j1) * v1;
1259  }
1260  }
1261  }
1263  //**********************************************************************************************
1264 
1265  //**Vectorized subtraction assignment to dense vectors******************************************
1279  template< typename VT1 // Type of the left-hand side target vector
1280  , typename MT1 // Type of the left-hand side matrix operand
1281  , typename VT2 > // Type of the right-hand side vector operand
1283  selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1284  {
1286 
1287  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1288 
1289  constexpr bool remainder( !IsPadded<MT1>::value || !IsPadded<VT1>::value );
1290 
1291  const size_t M( A.rows() );
1292 
1293  ConstIterator element( x.begin() );
1294  const ConstIterator end( x.end() );
1295 
1296  const size_t jpos( x.nonZeros() & size_t(-4) );
1297  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1298 
1299  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1300  {
1301  const size_t j1( element->index() );
1302  const VET v1( element->value() );
1303  ++element;
1304  const size_t j2( element->index() );
1305  const VET v2( element->value() );
1306  ++element;
1307  const size_t j3( element->index() );
1308  const VET v3( element->value() );
1309  ++element;
1310  const size_t j4( element->index() );
1311  const VET v4( element->value() );
1312  ++element;
1313 
1314  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1315 
1316  const SIMDType xmm1( set( v1 ) );
1317  const SIMDType xmm2( set( v2 ) );
1318  const SIMDType xmm3( set( v3 ) );
1319  const SIMDType xmm4( set( v4 ) );
1320 
1321  const size_t ibegin( ( IsLower<MT1>::value )
1322  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1323  :( 0UL ) );
1324  const size_t iend( ( IsUpper<MT1>::value )
1325  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
1326  :( M ) );
1327  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1328 
1329  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1330  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1331 
1332  size_t i( ibegin );
1333 
1334  for( ; i<ipos; i+=SIMDSIZE ) {
1335  y.store( i, y.load(i) - A.load(i,j1) * xmm1 - A.load(i,j2) * xmm2 - A.load(i,j3) * xmm3 - A.load(i,j4) * xmm4 );
1336  }
1337  for( ; remainder && i<iend; ++i ) {
1338  y[i] -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1339  }
1340  }
1341  for( ; element!=end; ++element )
1342  {
1343  const size_t j1( element->index() );
1344  const VET v1( element->value() );
1345 
1346  const SIMDType xmm1( set( v1 ) );
1347 
1348  const size_t ibegin( ( IsLower<MT1>::value )
1349  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1350  :( 0UL ) );
1351  const size_t iend( ( IsUpper<MT1>::value )
1352  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
1353  :( M ) );
1354  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1355 
1356  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1357  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1358 
1359  size_t i( ibegin );
1360 
1361  for( ; i<ipos; i+=SIMDSIZE ) {
1362  y.store( i, y.load(i) - A.load(i,j1) * xmm1 );
1363  }
1364  for( ; remainder && i<iend; ++i ) {
1365  y[i] -= A(i,j1) * v1;
1366  }
1367  }
1368  }
1370  //**********************************************************************************************
1371 
1372  //**Subtraction assignment to sparse vectors****************************************************
1373  // No special implementation for the subtraction assignment to sparse vectors.
1374  //**********************************************************************************************
1375 
1376  //**Multiplication assignment to dense vectors**************************************************
1389  template< typename VT1 > // Type of the target dense vector
1390  friend inline void multAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1391  {
1393 
1397 
1398  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1399 
1400  const ResultType tmp( serial( rhs ) );
1401  multAssign( ~lhs, tmp );
1402  }
1404  //**********************************************************************************************
1405 
1406  //**Multiplication assignment to sparse vectors*************************************************
1407  // No special implementation for the multiplication assignment to sparse vectors.
1408  //**********************************************************************************************
1409 
1410  //**Division assignment to dense vectors********************************************************
1423  template< typename VT1 > // Type of the target dense vector
1424  friend inline void divAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1425  {
1427 
1431 
1432  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1433 
1434  const ResultType tmp( serial( rhs ) );
1435  divAssign( ~lhs, tmp );
1436  }
1438  //**********************************************************************************************
1439 
1440  //**Division assignment to sparse vectors*******************************************************
1441  // No special implementation for the division assignment to sparse vectors.
1442  //**********************************************************************************************
1443 
1444  //**SMP assignment to dense vectors*************************************************************
1459  template< typename VT1 > // Type of the target dense vector
1460  friend inline EnableIf_< UseSMPAssign<VT1> >
1462  {
1464 
1465  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1466 
1467  // Evaluation of the right-hand side sparse vector operand
1468  RT x( rhs.vec_ );
1469  if( x.nonZeros() == 0UL ) {
1470  reset( ~lhs );
1471  return;
1472  }
1473 
1474  // Evaluation of the left-hand side dense matrix operand
1475  LT A( rhs.mat_ );
1476 
1477  // Checking the evaluated operands
1478  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1479  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1480  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1481  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1482 
1483  // Performing the dense matrix-sparse vector multiplication
1484  smpAssign( ~lhs, A * x );
1485  }
1487  //**********************************************************************************************
1488 
1489  //**SMP assignment to sparse vectors************************************************************
1504  template< typename VT1 > // Type of the target sparse vector
1505  friend inline EnableIf_< UseSMPAssign<VT1> >
1507  {
1509 
1513 
1514  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1515 
1516  const ResultType tmp( rhs );
1517  smpAssign( ~lhs, tmp );
1518  }
1520  //**********************************************************************************************
1521 
1522  //**SMP addition assignment to dense vectors****************************************************
1537  template< typename VT1 > // Type of the target dense vector
1538  friend inline EnableIf_< UseSMPAssign<VT1> >
1540  {
1542 
1543  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1544 
1545  // Evaluation of the right-hand side sparse vector operand
1546  RT x( rhs.vec_ );
1547  if( x.nonZeros() == 0UL ) return;
1548 
1549  // Evaluation of the left-hand side dense matrix operand
1550  LT A( rhs.mat_ );
1551 
1552  // Checking the evaluated operands
1553  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1554  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1555  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1556  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1557 
1558  // Performing the dense matrix-sparse vector multiplication
1559  smpAddAssign( ~lhs, A * x );
1560  }
1562  //**********************************************************************************************
1563 
1564  //**SMP addition assignment to sparse vectors***************************************************
1565  // No special implementation for the SMP addition assignment to sparse vectors.
1566  //**********************************************************************************************
1567 
1568  //**SMP subtraction assignment to dense vectors*************************************************
1583  template< typename VT1 > // Type of the target dense vector
1584  friend inline EnableIf_< UseSMPAssign<VT1> >
1586  {
1588 
1589  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1590 
1591  // Evaluation of the right-hand side sparse vector operand
1592  RT x( rhs.vec_ );
1593  if( x.nonZeros() == 0UL ) return;
1594 
1595  // Evaluation of the left-hand side dense matrix operand
1596  LT A( rhs.mat_ );
1597 
1598  // Checking the evaluated operands
1599  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1600  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1601  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1602  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1603 
1604  // Performing the dense matrix-sparse vector multiplication
1605  smpSubAssign( ~lhs, A * x );
1606  }
1608  //**********************************************************************************************
1609 
1610  //**SMP subtraction assignment to sparse vectors************************************************
1611  // No special implementation for the SMP subtraction assignment to sparse vectors.
1612  //**********************************************************************************************
1613 
1614  //**SMP multiplication assignment to dense vectors**********************************************
1629  template< typename VT1 > // Type of the target dense vector
1630  friend inline EnableIf_< UseSMPAssign<VT1> >
1632  {
1634 
1638 
1639  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1640 
1641  const ResultType tmp( rhs );
1642  smpMultAssign( ~lhs, tmp );
1643  }
1645  //**********************************************************************************************
1646 
1647  //**SMP multiplication assignment to sparse vectors*********************************************
1648  // No special implementation for the SMP multiplication assignment to sparse vectors.
1649  //**********************************************************************************************
1650 
1651  //**SMP division assignment to dense vectors****************************************************
1666  template< typename VT1 > // Type of the target dense vector
1667  friend inline EnableIf_< UseSMPAssign<VT1> >
1669  {
1671 
1675 
1676  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1677 
1678  const ResultType tmp( rhs );
1679  smpDivAssign( ~lhs, tmp );
1680  }
1682  //**********************************************************************************************
1683 
1684  //**SMP division assignment to sparse vectors***************************************************
1685  // No special implementation for the SMP division assignment to sparse vectors.
1686  //**********************************************************************************************
1687 
1688  //**Compile time checks*************************************************************************
1696  //**********************************************************************************************
1697 };
1698 //*************************************************************************************************
1699 
1700 
1701 
1702 
1703 //=================================================================================================
1704 //
1705 // GLOBAL BINARY ARITHMETIC OPERATORS
1706 //
1707 //=================================================================================================
1708 
1709 //*************************************************************************************************
1741 template< typename MT // Type of the left-hand side dense matrix
1742  , typename VT > // Type of the right-hand side sparse vector
1743 inline decltype(auto)
1744  operator*( const DenseMatrix<MT,true>& mat, const SparseVector<VT,false>& vec )
1745 {
1747 
1749 
1750  if( (~mat).columns() != (~vec).size() ) {
1751  BLAZE_THROW_INVALID_ARGUMENT( "Matrix and vector sizes do not match" );
1752  }
1753 
1754  using ReturnType = const TDMatSVecMultExpr<MT,VT>;
1755  return ReturnType( ~mat, ~vec );
1756 }
1757 //*************************************************************************************************
1758 
1759 
1760 
1761 
1762 //=================================================================================================
1763 //
1764 // SIZE SPECIALIZATIONS
1765 //
1766 //=================================================================================================
1767 
1768 //*************************************************************************************************
1770 template< typename MT, typename VT >
1771 struct Size< TDMatSVecMultExpr<MT,VT> >
1772  : public Rows<MT>
1773 {};
1775 //*************************************************************************************************
1776 
1777 
1778 
1779 
1780 //=================================================================================================
1781 //
1782 // ISALIGNED SPECIALIZATIONS
1783 //
1784 //=================================================================================================
1785 
1786 //*************************************************************************************************
1788 template< typename MT, typename VT >
1789 struct IsAligned< TDMatSVecMultExpr<MT,VT> >
1790  : public BoolConstant< IsAligned<MT>::value >
1791 {};
1793 //*************************************************************************************************
1794 
1795 } // namespace blaze
1796 
1797 #endif
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Header file for auxiliary alias declarations.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:72
RightOperand vec_
Right-hand side sparse vector of the multiplication expression.
Definition: TDMatSVecMultExpr.h:370
Header file for the Rows type trait.
Header file for basic type definitions.
MultTrait_< MRT, VRT > ResultType
Result type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:196
Subvector< VT, AF > subvector(Vector< VT, TF > &vector, size_t index, size_t size)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:322
If_< IsExpression< VT >, const VT, const VT &> RightOperand
Composite type of the right-hand side dense vector expression.
Definition: TDMatSVecMultExpr.h:207
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatSVecMultExpr.h:352
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a column dense or sparse vector type...
Definition: ColumnVector.h:61
EnableIf_< IsDenseMatrix< MT1 > > smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:164
Header file for the serial shim.
Header file for the IsDiagonal type trait.
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatSVecMultExpr.h:201
Generic wrapper for a compile time constant integral value.The IntegralConstant class template repres...
Definition: IntegralConstant.h:71
ResultType_< MT > MRT
Result type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:113
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the IsSame and IsStrictlySame type traits.
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:198
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:172
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:316
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:560
EnableIf_< IsDenseVector< VT1 > > smpMultAssign(Vector< VT1, TF1 > &lhs, const Vector< VT2, TF2 > &rhs)
Default implementation of the SMP multiplication assignment of a vector to a dense vector...
Definition: DenseVector.h:193
Header file for the DenseVector base class.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:88
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:171
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:250
Header file for the Computation base class.
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:88
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Compile time check for data types.This type trait tests whether or not the given types can be combine...
Definition: IsSIMDCombinable.h:120
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:343
EnableIf_< IsDenseMatrix< MT1 > > smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:133
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:78
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
Constraint on the transpose flag of vector types.
ReturnType at(size_t index) const
Checked access to the vector elements.
Definition: TDMatSVecMultExpr.h:285
Row< MT > row(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific row of the given matrix.
Definition: Row.h:124
Compile time check for the alignment of data types.This type trait tests whether the given data type ...
Definition: IsAligned.h:87
Constraint on the data type.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:72
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
Header file for the DisableIf class template.
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:197
Header file for the If class template.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
Compile time check for data types with padding.This type trait tests whether the given data type empl...
Definition: IsPadded.h:76
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:102
LeftOperand mat_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatSVecMultExpr.h:369
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:3087
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
Base class for N-dimensional dense vectors.The DenseVector class is a base class for all arbitrarily ...
Definition: DenseVector.h:70
Header file for the IsLower type trait.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional vector type...
Definition: SparseVector.h:61
CompositeType_< VT > VCT
Composite type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:118
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:90
BLAZE_ALWAYS_INLINE size_t columns(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of columns of the matrix.
Definition: Matrix.h:340
#define BLAZE_CONSTRAINT_MUST_NOT_BE_MATMATMULTEXPR_TYPE(T)
Constraint on the data type.In case the given data type T is a matrix/matrix multiplication expressio...
Definition: MatMatMultExpr.h:88
LeftOperand leftOperand() const noexcept
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatSVecMultExpr.h:308
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATVECMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/vector ...
Definition: MatVecMultExpr.h:109
Constraint on the data type.
Header file for the exception macros of the math module.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:264
ReturnType operator[](size_t index) const
Subscript operator for the direct access to the vector elements.
Definition: TDMatSVecMultExpr.h:253
ResultType_< VT > VRT
Result type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:114
Constraint on the data type.
Header file for all forward declarations for expression class templates.
Constraint on the data type.
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
RightOperand rightOperand() const noexcept
Returns the right-hand side sparse vector operand.
Definition: TDMatSVecMultExpr.h:318
Expression object for transpose dense matrix-sparse vector multiplications.The TDMatSVecMultExpr clas...
Definition: Forward.h:153
Compile time check for resizable data types.This type trait tests whether the given data type is a re...
Definition: IsResizable.h:75
Header file for the IsSIMDCombinable type trait.
Header file for the HasSIMDMult type trait.
Header file for run time assertion macros.
size_t size() const noexcept
Returns the current size/dimension of the vector.
Definition: TDMatSVecMultExpr.h:298
If_< IsExpression< MT >, const MT, const MT &> LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:204
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:154
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TDMatSVecMultExpr.h:330
ElementType_< MRT > MET
Element type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:115
EnableIf_< IsDenseVector< VT1 > > smpDivAssign(Vector< VT1, TF1 > &lhs, const Vector< VT2, TF2 > &rhs)
Default implementation of the SMP division assignment of a vector to a dense vector.
Definition: DenseVector.h:222
Header file for the reset shim.
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:296
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:94
Constraints on the storage order of matrix types.
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:819
TDMatSVecMultExpr(const MT &mat, const VT &vec) noexcept
Constructor for the TDMatSVecMultExpr class.
Definition: TDMatSVecMultExpr.h:239
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
CompositeType_< MT > MCT
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:117
Header file for the RemoveReference type trait.
ElementType_< VRT > VET
Element type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:116
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:224
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
const Type & ReturnType
Return type for expression template evaluations.
Definition: CompressedMatrix.h:3082
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional vector type...
Definition: DenseVector.h:61
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: TDMatSVecMultExpr.h:199
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TDMatSVecMultExpr.h:342
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TDMatSVecMultExpr.h:362
Header file for the IsComputation type trait class.
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:200
IfTrue_< evaluateVector, const VRT, CompositeType_< VT > > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: TDMatSVecMultExpr.h:213
Compile time evaluation of the size of a vector.The Size type trait evaluates the size of the given v...
Definition: Size.h:74
Base class for sparse vectors.The SparseVector class is a base class for all arbitrarily sized (N-dim...
Definition: Forward.h:130
Header file for the IntegralConstant class template.
ElementType_< ResultType > ElementType
Resulting element type.
Definition: TDMatSVecMultExpr.h:198
Compile time evaluation of the number of rows of a matrix.The Rows type trait evaluates the number of...
Definition: Rows.h:75
Constraint on the data type.
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:423
Header file for the IsUpper type trait.
Header file for the MatVecMultExpr base class.
Constraint on the data type.
Header file for the IsResizable type trait.
Header file for the Size type trait.
Size type of the Blaze library.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
IfTrue_< evaluateMatrix, const MRT, MCT > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatSVecMultExpr.h:210
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the IsExpression type trait class.
Header file for the function trace functionality.