TDMatSVecMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TDMATSVECMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TDMATSVECMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
53 #include <blaze/math/Exception.h>
58 #include <blaze/math/shims/Reset.h>
60 #include <blaze/math/SIMD.h>
77 #include <blaze/math/views/Check.h>
80 #include <blaze/util/Assert.h>
81 #include <blaze/util/DisableIf.h>
82 #include <blaze/util/EnableIf.h>
84 #include <blaze/util/mpl/If.h>
85 #include <blaze/util/Types.h>
88 
89 
90 namespace blaze {
91 
92 //=================================================================================================
93 //
94 // CLASS TDMATSVECMULTEXPR
95 //
96 //=================================================================================================
97 
98 //*************************************************************************************************
105 template< typename MT // Type of the left-hand side dense matrix
106  , typename VT > // Type of the right-hand side sparse vector
107 class TDMatSVecMultExpr
108  : public MatVecMultExpr< DenseVector< TDMatSVecMultExpr<MT,VT>, false > >
109  , private Computation
110 {
111  private:
112  //**Type definitions****************************************************************************
119  //**********************************************************************************************
120 
121  //**********************************************************************************************
123  enum : bool { evaluateMatrix = RequiresEvaluation<MT>::value };
124  //**********************************************************************************************
125 
126  //**********************************************************************************************
128  enum : bool { evaluateVector = IsComputation<VT>::value || RequiresEvaluation<VT>::value };
129  //**********************************************************************************************
130 
131  //**********************************************************************************************
133 
137  template< typename T1 >
138  struct UseSMPAssign {
139  enum : bool { value = ( evaluateMatrix || evaluateVector ) };
140  };
142  //**********************************************************************************************
143 
144  //**********************************************************************************************
146 
150  template< typename T1, typename T2, typename T3 >
151  struct UseVectorizedKernel {
152  enum : bool { value = useOptimizedKernels &&
154  T1::simdEnabled && T2::simdEnabled &&
157  , ElementType_<T3> >::value &&
160  };
162  //**********************************************************************************************
163 
164  //**********************************************************************************************
166 
170  template< typename T1, typename T2, typename T3 >
171  struct UseOptimizedKernel {
172  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
174  !IsResizable< ElementType_<T1> >::value &&
176  };
178  //**********************************************************************************************
179 
180  //**********************************************************************************************
182 
185  template< typename T1, typename T2, typename T3 >
186  struct UseDefaultKernel {
187  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
188  !UseOptimizedKernel<T1,T2,T3>::value };
189  };
191  //**********************************************************************************************
192 
193  public:
194  //**Type definitions****************************************************************************
200  using ReturnType = const ElementType;
201  using CompositeType = const ResultType;
202 
204  using LeftOperand = If_< IsExpression<MT>, const MT, const MT& >;
205 
207  using RightOperand = If_< IsExpression<VT>, const VT, const VT& >;
208 
211 
214  //**********************************************************************************************
215 
216  //**Compilation flags***************************************************************************
218  enum : bool { simdEnabled = !IsDiagonal<MT>::value &&
219  MT::simdEnabled &&
222 
224  enum : bool { smpAssignable = !evaluateMatrix && MT::smpAssignable &&
225  !evaluateVector && VT::smpAssignable };
226  //**********************************************************************************************
227 
228  //**SIMD properties*****************************************************************************
230  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
231  //**********************************************************************************************
232 
233  //**Constructor*********************************************************************************
239  explicit inline TDMatSVecMultExpr( const MT& mat, const VT& vec ) noexcept
240  : mat_( mat ) // Left-hand side dense matrix of the multiplication expression
241  , vec_( vec ) // Right-hand side sparse vector of the multiplication expression
242  {
243  BLAZE_INTERNAL_ASSERT( mat_.columns() == vec_.size(), "Invalid matrix and vector sizes" );
244  }
245  //**********************************************************************************************
246 
247  //**Subscript operator**************************************************************************
253  inline ReturnType operator[]( size_t index ) const {
254  BLAZE_INTERNAL_ASSERT( index < mat_.rows(), "Invalid vector access index" );
255 
257  {
258  return mat_(index,index) * vec_[index];
259  }
260  else if( IsLower<MT>::value )
261  {
262  const size_t n( IsStrictlyLower<MT>::value ? index : index+1UL );
263  return subvector( row( mat_, index, unchecked ), 0UL, n, unchecked ) *
264  subvector( vec_, 0UL, n, unchecked );
265  }
266  else if( IsUpper<MT>::value )
267  {
268  const size_t begin( IsStrictlyUpper<MT>::value ? index+1UL : index );
269  const size_t n ( mat_.columns() - begin );
270  return subvector( row( mat_, index, unchecked ), begin, n, unchecked ) *
271  subvector( vec_, begin, n, unchecked );
272  }
273  else
274  {
275  return row( mat_, index, unchecked ) * vec_;
276  }
277  }
278  //**********************************************************************************************
279 
280  //**At function*********************************************************************************
287  inline ReturnType at( size_t index ) const {
288  if( index >= mat_.rows() ) {
289  BLAZE_THROW_OUT_OF_RANGE( "Invalid vector access index" );
290  }
291  return (*this)[index];
292  }
293  //**********************************************************************************************
294 
295  //**Size function*******************************************************************************
300  inline size_t size() const noexcept {
301  return mat_.rows();
302  }
303  //**********************************************************************************************
304 
305  //**Left operand access*************************************************************************
310  inline LeftOperand leftOperand() const noexcept {
311  return mat_;
312  }
313  //**********************************************************************************************
314 
315  //**Right operand access************************************************************************
320  inline RightOperand rightOperand() const noexcept {
321  return vec_;
322  }
323  //**********************************************************************************************
324 
325  //**********************************************************************************************
331  template< typename T >
332  inline bool canAlias( const T* alias ) const noexcept {
333  return mat_.isAliased( alias ) || vec_.isAliased( alias );
334  }
335  //**********************************************************************************************
336 
337  //**********************************************************************************************
343  template< typename T >
344  inline bool isAliased( const T* alias ) const noexcept {
345  return mat_.isAliased( alias ) || vec_.isAliased( alias );
346  }
347  //**********************************************************************************************
348 
349  //**********************************************************************************************
354  inline bool isAligned() const noexcept {
355  return mat_.isAligned();
356  }
357  //**********************************************************************************************
358 
359  //**********************************************************************************************
364  inline bool canSMPAssign() const noexcept {
365  return ( size() > SMP_TDMATSVECMULT_THRESHOLD );
366  }
367  //**********************************************************************************************
368 
369  private:
370  //**Member variables****************************************************************************
373  //**********************************************************************************************
374 
375  //**Assignment to dense vectors*****************************************************************
388  template< typename VT1 > // Type of the target dense vector
389  friend inline void assign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
390  {
392 
393  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
394 
395  // Evaluation of the right-hand side sparse vector operand
396  RT x( serial( rhs.vec_ ) );
397  if( x.nonZeros() == 0UL ) {
398  reset( ~lhs );
399  return;
400  }
401 
402  // Evaluation of the left-hand side dense matrix operand
403  LT A( serial( rhs.mat_ ) );
404 
405  // Checking the evaluated operands
406  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
407  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
408  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
409  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
410 
411  // Performing the dense matrix-sparse vector multiplication
412  TDMatSVecMultExpr::selectAssignKernel( ~lhs, A, x );
413  }
415  //**********************************************************************************************
416 
417  //**Default assignment to dense vectors*********************************************************
431  template< typename VT1 // Type of the left-hand side target vector
432  , typename MT1 // Type of the left-hand side matrix operand
433  , typename VT2 > // Type of the right-hand side vector operand
435  selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
436  {
438 
439  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
440 
441  const size_t M( A.rows() );
442 
443  ConstIterator element( x.begin() );
444  const ConstIterator end( x.end() );
445 
446  size_t last( 0UL );
447 
448  if( IsLower<MT1>::value ) {
449  const size_t iend( IsStrictlyLower<MT1>::value ? element->index()+1UL : element->index() );
450  for( size_t i=0UL; i<iend; ++i )
451  reset( y[i] );
452  }
453 
454  for( ; element!=end; ++element )
455  {
456  const size_t index( element->index() );
457 
459  {
460  for( size_t i=last; i<index; ++i )
461  reset( y[i] );
462 
463  y[index] = A(index,index) * element->value();
464  last = index + 1UL;
465  }
466  else
467  {
468  const size_t ibegin( ( IsLower<MT1>::value )
469  ?( IsStrictlyLower<MT1>::value ? index+1UL : index )
470  :( 0UL ) );
471  const size_t iend( ( IsUpper<MT1>::value )
472  ?( IsStrictlyUpper<MT1>::value ? index : index+1UL )
473  :( M ) );
474  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
475 
476  for( size_t i=ibegin; i<last; ++i ) {
477  y[i] += A(i,index) * element->value();
478  }
479  for( size_t i=last; i<iend; ++i ) {
480  y[i] = A(i,index) * element->value();
481  }
482 
483  last = iend;
484  }
485  }
486 
487  if( IsUpper<MT1>::value ) {
488  for( size_t i=last; i<M; ++i )
489  reset( y[i] );
490  }
491  }
493  //**********************************************************************************************
494 
495  //**Optimized assignment to dense vectors*******************************************************
509  template< typename VT1 // Type of the left-hand side target vector
510  , typename MT1 // Type of the left-hand side matrix operand
511  , typename VT2 > // Type of the right-hand side vector operand
513  selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
514  {
516 
517  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
518 
519  const size_t M( A.rows() );
520 
521  ConstIterator element( x.begin() );
522  const ConstIterator end( x.end() );
523 
524  const size_t jpos( x.nonZeros() & size_t(-4) );
525  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
526 
527  if( jpos > 3UL )
528  {
529  const size_t j1( element->index() );
530  const VET v1( element->value() );
531  ++element;
532  const size_t j2( element->index() );
533  const VET v2( element->value() );
534  ++element;
535  const size_t j3( element->index() );
536  const VET v3( element->value() );
537  ++element;
538  const size_t j4( element->index() );
539  const VET v4( element->value() );
540  ++element;
541 
542  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
543 
544  for( size_t i=0UL; i<M; ++i ) {
545  y[i] = A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
546  }
547  }
548  else
549  {
550  const size_t j1( element->index() );
551  const VET v1( element->value() );
552  ++element;
553 
554  for( size_t i=0UL; i<M; ++i ) {
555  y[i] = A(i,j1) * v1;
556  }
557  }
558 
559  for( size_t j=(jpos>3UL)?(4UL):(1UL); (j+4UL)<=jpos; j+=4UL )
560  {
561  const size_t j1( element->index() );
562  const VET v1( element->value() );
563  ++element;
564  const size_t j2( element->index() );
565  const VET v2( element->value() );
566  ++element;
567  const size_t j3( element->index() );
568  const VET v3( element->value() );
569  ++element;
570  const size_t j4( element->index() );
571  const VET v4( element->value() );
572  ++element;
573 
574  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
575 
576  const size_t ibegin( ( IsLower<MT1>::value )
577  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
578  :( 0UL ) );
579  const size_t iend( ( IsUpper<MT1>::value )
580  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
581  :( M ) );
582  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
583 
584  for( size_t i=ibegin; i<iend; ++i ) {
585  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
586  }
587  }
588  for( ; element!=end; ++element )
589  {
590  const size_t j1( element->index() );
591  const VET v1( element->value() );
592 
593  const size_t ibegin( ( IsLower<MT1>::value )
594  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
595  :( 0UL ) );
596  const size_t iend( ( IsUpper<MT1>::value )
597  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
598  :( M ) );
599  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
600 
601  for( size_t i=ibegin; i<iend; ++i ) {
602  y[i] += A(i,j1) * v1;
603  }
604  }
605  }
607  //**********************************************************************************************
608 
609  //**Vectorized assignment to dense vectors******************************************************
623  template< typename VT1 // Type of the left-hand side target vector
624  , typename MT1 // Type of the left-hand side matrix operand
625  , typename VT2 > // Type of the right-hand side vector operand
627  selectAssignKernel( VT1& y, const MT1& A, const VT2& x )
628  {
630 
631  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
632 
633  constexpr bool remainder( !IsPadded<MT1>::value || !IsPadded<VT1>::value );
634 
635  const size_t M( A.rows() );
636 
637  ConstIterator element( x.begin() );
638  const ConstIterator end( x.end() );
639 
640  const size_t jpos( x.nonZeros() & size_t(-4) );
641  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
642 
643  if( jpos > 3UL )
644  {
645  const size_t j1( element->index() );
646  const VET v1( element->value() );
647  ++element;
648  const size_t j2( element->index() );
649  const VET v2( element->value() );
650  ++element;
651  const size_t j3( element->index() );
652  const VET v3( element->value() );
653  ++element;
654  const size_t j4( element->index() );
655  const VET v4( element->value() );
656  ++element;
657 
658  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
659 
660  const SIMDType xmm1( set( v1 ) );
661  const SIMDType xmm2( set( v2 ) );
662  const SIMDType xmm3( set( v3 ) );
663  const SIMDType xmm4( set( v4 ) );
664 
665  const size_t ipos( remainder ? ( M & size_t(-SIMDSIZE) ) : M );
666  BLAZE_INTERNAL_ASSERT( !remainder || ( M - ( M % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
667 
668  size_t i( 0UL );
669 
670  for( ; i<ipos; i+=SIMDSIZE ) {
671  y.store( i, A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
672  }
673  for( ; remainder && i<M; ++i ) {
674  y[i] = A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
675  }
676  }
677  else
678  {
679  const size_t j1( element->index() );
680  const VET v1( element->value() );
681  ++element;
682 
683  const SIMDType xmm1( set( v1 ) );
684 
685  const size_t ipos( remainder ? ( M & size_t(-SIMDSIZE) ) : M );
686  BLAZE_INTERNAL_ASSERT( !remainder || ( M - ( M % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
687 
688  size_t i( 0UL );
689 
690  for( ; i<ipos; i+=SIMDSIZE ) {
691  y.store( i, A.load(i,j1) * xmm1 );
692  }
693  for( ; remainder && i<M; ++i ) {
694  y[i] = A(i,j1) * v1;
695  }
696  }
697 
698  for( size_t j=(jpos>3UL)?(4UL):(1UL); (j+4UL)<=jpos; j+=4UL )
699  {
700  const size_t j1( element->index() );
701  const VET v1( element->value() );
702  ++element;
703  const size_t j2( element->index() );
704  const VET v2( element->value() );
705  ++element;
706  const size_t j3( element->index() );
707  const VET v3( element->value() );
708  ++element;
709  const size_t j4( element->index() );
710  const VET v4( element->value() );
711  ++element;
712 
713  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
714 
715  const SIMDType xmm1( set( v1 ) );
716  const SIMDType xmm2( set( v2 ) );
717  const SIMDType xmm3( set( v3 ) );
718  const SIMDType xmm4( set( v4 ) );
719 
720  const size_t ibegin( ( IsLower<MT1>::value )
721  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
722  :( 0UL ) );
723  const size_t iend( ( IsUpper<MT1>::value )
724  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
725  :( M ) );
726  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
727 
728  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
729  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
730 
731  size_t i( ibegin );
732 
733  for( ; i<ipos; i+=SIMDSIZE ) {
734  y.store( i, y.load(i) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
735  }
736  for( ; remainder && i<iend; ++i ) {
737  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
738  }
739  }
740 
741  for( ; element!=end; ++element )
742  {
743  const size_t j1( element->index() );
744  const VET v1( element->value() );
745 
746  const SIMDType xmm1( set( v1 ) );
747 
748  const size_t ibegin( ( IsLower<MT1>::value )
749  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
750  :( 0UL ) );
751  const size_t iend( ( IsUpper<MT1>::value )
752  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
753  :( M ) );
754  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
755 
756  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
757  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
758 
759  size_t i( ibegin );
760 
761  for( ; i<ipos; i+=SIMDSIZE ) {
762  y.store( i, y.load(i) + A.load(i,j1) * xmm1 );
763  }
764  for( ; remainder && i<iend; ++i ) {
765  y[i] += A(i,j1) * v1;
766  }
767  }
768  }
770  //**********************************************************************************************
771 
772  //**Assignment to sparse vectors****************************************************************
785  template< typename VT1 > // Type of the target sparse vector
786  friend inline void assign( SparseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
787  {
789 
793 
794  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
795 
796  const ResultType tmp( serial( rhs ) );
797  assign( ~lhs, tmp );
798  }
800  //**********************************************************************************************
801 
802  //**Addition assignment to dense vectors********************************************************
815  template< typename VT1 > // Type of the target dense vector
816  friend inline void addAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
817  {
819 
820  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
821 
822  // Evaluation of the right-hand side sparse vector operand
823  RT x( serial( rhs.vec_ ) );
824  if( x.nonZeros() == 0UL ) return;
825 
826  // Evaluation of the left-hand side dense matrix operand
827  LT A( serial( rhs.mat_ ) );
828 
829  // Checking the evaluated operands
830  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
831  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
832  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
833  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
834 
835  // Performing the dense matrix-sparse vector multiplication
836  TDMatSVecMultExpr::selectAddAssignKernel( ~lhs, A, x );
837  }
839  //**********************************************************************************************
840 
841  //**Default addition assignment to dense vectors************************************************
855  template< typename VT1 // Type of the left-hand side target vector
856  , typename MT1 // Type of the left-hand side matrix operand
857  , typename VT2 > // Type of the right-hand side vector operand
859  selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
860  {
862 
863  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
864 
865  const size_t M( A.rows() );
866 
867  ConstIterator element( x.begin() );
868  const ConstIterator end( x.end() );
869 
870  for( ; element!=end; ++element )
871  {
872  const size_t index( element->index() );
873 
875  {
876  y[index] += A(index,index) * element->value();
877  }
878  else
879  {
880  const size_t ibegin( ( IsLower<MT1>::value )
881  ?( IsStrictlyLower<MT1>::value ? index+1UL : index )
882  :( 0UL ) );
883  const size_t iend( ( IsUpper<MT1>::value )
884  ?( IsStrictlyUpper<MT1>::value ? index : index+1UL )
885  :( M ) );
886  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
887 
888  for( size_t i=ibegin; i<iend; ++i ) {
889  y[i] += A(i,index) * element->value();
890  }
891  }
892  }
893  }
895  //**********************************************************************************************
896 
897  //**Optimized addition assignment to dense vectors**********************************************
911  template< typename VT1 // Type of the left-hand side target vector
912  , typename MT1 // Type of the left-hand side matrix operand
913  , typename VT2 > // Type of the right-hand side vector operand
915  selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
916  {
918 
919  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
920 
921  const size_t M( A.rows() );
922 
923  ConstIterator element( x.begin() );
924  const ConstIterator end( x.end() );
925 
926  const size_t jpos( x.nonZeros() & size_t(-4) );
927  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
928 
929  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
930  {
931  const size_t j1( element->index() );
932  const VET v1( element->value() );
933  ++element;
934  const size_t j2( element->index() );
935  const VET v2( element->value() );
936  ++element;
937  const size_t j3( element->index() );
938  const VET v3( element->value() );
939  ++element;
940  const size_t j4( element->index() );
941  const VET v4( element->value() );
942  ++element;
943 
944  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
945 
946  const size_t ibegin( ( IsLower<MT1>::value )
947  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
948  :( 0UL ) );
949  const size_t iend( ( IsUpper<MT1>::value )
950  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
951  :( M ) );
952  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
953 
954  for( size_t i=ibegin; i<iend; ++i ) {
955  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
956  }
957  }
958  for( ; element!=end; ++element )
959  {
960  const size_t j1( element->index() );
961  const VET v1( element->value() );
962 
963  const size_t ibegin( ( IsLower<MT1>::value )
964  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
965  :( 0UL ) );
966  const size_t iend( ( IsUpper<MT1>::value )
967  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
968  :( M ) );
969  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
970 
971  for( size_t i=ibegin; i<iend; ++i ) {
972  y[i] += A(i,j1) * v1;
973  }
974  }
975  }
977  //**********************************************************************************************
978 
979  //**Vectorized addition assignment to dense vectors*********************************************
993  template< typename VT1 // Type of the left-hand side target vector
994  , typename MT1 // Type of the left-hand side matrix operand
995  , typename VT2 > // Type of the right-hand side vector operand
997  selectAddAssignKernel( VT1& y, const MT1& A, const VT2& x )
998  {
1000 
1001  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1002 
1003  constexpr bool remainder( !IsPadded<MT1>::value || !IsPadded<VT1>::value );
1004 
1005  const size_t M( A.rows() );
1006 
1007  ConstIterator element( x.begin() );
1008  const ConstIterator end( x.end() );
1009 
1010  const size_t jpos( x.nonZeros() & size_t(-4) );
1011  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1012 
1013  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1014  {
1015  const size_t j1( element->index() );
1016  const VET v1( element->value() );
1017  ++element;
1018  const size_t j2( element->index() );
1019  const VET v2( element->value() );
1020  ++element;
1021  const size_t j3( element->index() );
1022  const VET v3( element->value() );
1023  ++element;
1024  const size_t j4( element->index() );
1025  const VET v4( element->value() );
1026  ++element;
1027 
1028  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1029 
1030  const SIMDType xmm1( set( v1 ) );
1031  const SIMDType xmm2( set( v2 ) );
1032  const SIMDType xmm3( set( v3 ) );
1033  const SIMDType xmm4( set( v4 ) );
1034 
1035  const size_t ibegin( ( IsLower<MT1>::value )
1036  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1037  :( 0UL ) );
1038  const size_t iend( ( IsUpper<MT1>::value )
1039  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
1040  :( M ) );
1041  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1042 
1043  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1044  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1045 
1046  size_t i( ibegin );
1047 
1048  for( ; i<ipos; i+=SIMDSIZE ) {
1049  y.store( i, y.load(i) + A.load(i,j1) * xmm1 + A.load(i,j2) * xmm2 + A.load(i,j3) * xmm3 + A.load(i,j4) * xmm4 );
1050  }
1051  for( ; remainder && i<iend; ++i ) {
1052  y[i] += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1053  }
1054  }
1055  for( ; element!=end; ++element )
1056  {
1057  const size_t j1( element->index() );
1058  const VET v1( element->value() );
1059 
1060  const SIMDType xmm1( set( v1 ) );
1061 
1062  const size_t ibegin( ( IsLower<MT1>::value )
1063  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1064  :( 0UL ) );
1065  const size_t iend( ( IsUpper<MT1>::value )
1066  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
1067  :( M ) );
1068  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1069 
1070  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1071  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1072 
1073  size_t i( ibegin );
1074 
1075  for( ; i<ipos; i+=SIMDSIZE ) {
1076  y.store( i, y.load(i) + A.load(i,j1) * xmm1 );
1077  }
1078  for( ; remainder && i<iend; ++i ) {
1079  y[i] += A(i,j1) * v1;
1080  }
1081  }
1082  }
1084  //**********************************************************************************************
1085 
1086  //**Addition assignment to sparse vectors*******************************************************
1087  // No special implementation for the addition assignment to sparse vectors.
1088  //**********************************************************************************************
1089 
1090  //**Subtraction assignment to dense vectors*****************************************************
1103  template< typename VT1 > // Type of the target dense vector
1104  friend inline void subAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1105  {
1107 
1108  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1109 
1110  // Evaluation of the right-hand side sparse vector operand
1111  RT x( serial( rhs.vec_ ) );
1112  if( x.nonZeros() == 0UL ) return;
1113 
1114  // Evaluation of the left-hand side dense matrix operand
1115  LT A( serial( rhs.mat_ ) );
1116 
1117  // Checking the evaluated operands
1118  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1119  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1120  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1121  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1122 
1123  // Performing the dense matrix-sparse vector multiplication
1124  TDMatSVecMultExpr::selectSubAssignKernel( ~lhs, A, x );
1125  }
1127  //**********************************************************************************************
1128 
1129  //**Default subtraction assignment to dense vectors*********************************************
1143  template< typename VT1 // Type of the left-hand side target vector
1144  , typename MT1 // Type of the left-hand side matrix operand
1145  , typename VT2 > // Type of the right-hand side vector operand
1147  selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1148  {
1150 
1151  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1152 
1153  const size_t M( A.rows() );
1154 
1155  ConstIterator element( x.begin() );
1156  const ConstIterator end( x.end() );
1157 
1158  for( ; element!=end; ++element )
1159  {
1160  const size_t index( element->index() );
1161 
1163  {
1164  y[index] -= A(index,index) * element->value();
1165  }
1166  else
1167  {
1168  const size_t ibegin( ( IsLower<MT1>::value )
1169  ?( IsStrictlyLower<MT1>::value ? index+1UL : index )
1170  :( 0UL ) );
1171  const size_t iend( ( IsUpper<MT1>::value )
1172  ?( IsStrictlyUpper<MT1>::value ? index : index+1UL )
1173  :( M ) );
1174  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1175 
1176  for( size_t i=ibegin; i<iend; ++i ) {
1177  y[i] -= A(i,index) * element->value();
1178  }
1179  }
1180  }
1181  }
1183  //**********************************************************************************************
1184 
1185  //**Optimized subtraction assignment to dense vectors*******************************************
1199  template< typename VT1 // Type of the left-hand side target vector
1200  , typename MT1 // Type of the left-hand side matrix operand
1201  , typename VT2 > // Type of the right-hand side vector operand
1203  selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1204  {
1206 
1207  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1208 
1209  const size_t M( A.rows() );
1210 
1211  ConstIterator element( x.begin() );
1212  const ConstIterator end( x.end() );
1213 
1214  const size_t jpos( x.nonZeros() & size_t(-4) );
1215  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1216 
1217  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1218  {
1219  const size_t j1( element->index() );
1220  const VET v1( element->value() );
1221  ++element;
1222  const size_t j2( element->index() );
1223  const VET v2( element->value() );
1224  ++element;
1225  const size_t j3( element->index() );
1226  const VET v3( element->value() );
1227  ++element;
1228  const size_t j4( element->index() );
1229  const VET v4( element->value() );
1230  ++element;
1231 
1232  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1233 
1234  const size_t ibegin( ( IsLower<MT1>::value )
1235  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
1236  :( 0UL ) );
1237  const size_t iend( ( IsUpper<MT1>::value )
1238  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
1239  :( M ) );
1240  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1241 
1242  for( size_t i=ibegin; i<iend; ++i ) {
1243  y[i] -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1244  }
1245  }
1246  for( ; element!=end; ++element )
1247  {
1248  const size_t j1( element->index() );
1249  const VET v1( element->value() );
1250 
1251  const size_t ibegin( ( IsLower<MT1>::value )
1252  ?( IsStrictlyLower<MT1>::value ? j1+1UL : j1 )
1253  :( 0UL ) );
1254  const size_t iend( ( IsUpper<MT1>::value )
1255  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
1256  :( M ) );
1257  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1258 
1259  for( size_t i=ibegin; i<iend; ++i ) {
1260  y[i] -= A(i,j1) * v1;
1261  }
1262  }
1263  }
1265  //**********************************************************************************************
1266 
1267  //**Vectorized subtraction assignment to dense vectors******************************************
1281  template< typename VT1 // Type of the left-hand side target vector
1282  , typename MT1 // Type of the left-hand side matrix operand
1283  , typename VT2 > // Type of the right-hand side vector operand
1285  selectSubAssignKernel( VT1& y, const MT1& A, const VT2& x )
1286  {
1288 
1289  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1290 
1291  constexpr bool remainder( !IsPadded<MT1>::value || !IsPadded<VT1>::value );
1292 
1293  const size_t M( A.rows() );
1294 
1295  ConstIterator element( x.begin() );
1296  const ConstIterator end( x.end() );
1297 
1298  const size_t jpos( x.nonZeros() & size_t(-4) );
1299  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == jpos, "Invalid end calculation" );
1300 
1301  for( size_t j=0UL; (j+4UL)<=jpos; j+=4UL )
1302  {
1303  const size_t j1( element->index() );
1304  const VET v1( element->value() );
1305  ++element;
1306  const size_t j2( element->index() );
1307  const VET v2( element->value() );
1308  ++element;
1309  const size_t j3( element->index() );
1310  const VET v3( element->value() );
1311  ++element;
1312  const size_t j4( element->index() );
1313  const VET v4( element->value() );
1314  ++element;
1315 
1316  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse vector index detected" );
1317 
1318  const SIMDType xmm1( set( v1 ) );
1319  const SIMDType xmm2( set( v2 ) );
1320  const SIMDType xmm3( set( v3 ) );
1321  const SIMDType xmm4( set( v4 ) );
1322 
1323  const size_t ibegin( ( IsLower<MT1>::value )
1324  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1325  :( 0UL ) );
1326  const size_t iend( ( IsUpper<MT1>::value )
1327  ?( IsStrictlyUpper<MT1>::value ? j4 : j4+1UL )
1328  :( M ) );
1329  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1330 
1331  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1332  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1333 
1334  size_t i( ibegin );
1335 
1336  for( ; i<ipos; i+=SIMDSIZE ) {
1337  y.store( i, y.load(i) - A.load(i,j1) * xmm1 - A.load(i,j2) * xmm2 - A.load(i,j3) * xmm3 - A.load(i,j4) * xmm4 );
1338  }
1339  for( ; remainder && i<iend; ++i ) {
1340  y[i] -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1341  }
1342  }
1343  for( ; element!=end; ++element )
1344  {
1345  const size_t j1( element->index() );
1346  const VET v1( element->value() );
1347 
1348  const SIMDType xmm1( set( v1 ) );
1349 
1350  const size_t ibegin( ( IsLower<MT1>::value )
1351  ?( ( IsStrictlyLower<MT1>::value ? j1+1UL : j1 ) & size_t(-SIMDSIZE) )
1352  :( 0UL ) );
1353  const size_t iend( ( IsUpper<MT1>::value )
1354  ?( IsStrictlyUpper<MT1>::value ? j1 : j1+1UL )
1355  :( M ) );
1356  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1357 
1358  const size_t ipos( remainder ? ( iend & size_t(-SIMDSIZE) ) : iend );
1359  BLAZE_INTERNAL_ASSERT( !remainder || ( iend - ( iend % SIMDSIZE ) ) == ipos, "Invalid end calculation" );
1360 
1361  size_t i( ibegin );
1362 
1363  for( ; i<ipos; i+=SIMDSIZE ) {
1364  y.store( i, y.load(i) - A.load(i,j1) * xmm1 );
1365  }
1366  for( ; remainder && i<iend; ++i ) {
1367  y[i] -= A(i,j1) * v1;
1368  }
1369  }
1370  }
1372  //**********************************************************************************************
1373 
1374  //**Subtraction assignment to sparse vectors****************************************************
1375  // No special implementation for the subtraction assignment to sparse vectors.
1376  //**********************************************************************************************
1377 
1378  //**Multiplication assignment to dense vectors**************************************************
1391  template< typename VT1 > // Type of the target dense vector
1392  friend inline void multAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1393  {
1395 
1399 
1400  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1401 
1402  const ResultType tmp( serial( rhs ) );
1403  multAssign( ~lhs, tmp );
1404  }
1406  //**********************************************************************************************
1407 
1408  //**Multiplication assignment to sparse vectors*************************************************
1409  // No special implementation for the multiplication assignment to sparse vectors.
1410  //**********************************************************************************************
1411 
1412  //**Division assignment to dense vectors********************************************************
1425  template< typename VT1 > // Type of the target dense vector
1426  friend inline void divAssign( DenseVector<VT1,false>& lhs, const TDMatSVecMultExpr& rhs )
1427  {
1429 
1433 
1434  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1435 
1436  const ResultType tmp( serial( rhs ) );
1437  divAssign( ~lhs, tmp );
1438  }
1440  //**********************************************************************************************
1441 
1442  //**Division assignment to sparse vectors*******************************************************
1443  // No special implementation for the division assignment to sparse vectors.
1444  //**********************************************************************************************
1445 
1446  //**SMP assignment to dense vectors*************************************************************
1461  template< typename VT1 > // Type of the target dense vector
1462  friend inline EnableIf_< UseSMPAssign<VT1> >
1464  {
1466 
1467  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1468 
1469  // Evaluation of the right-hand side sparse vector operand
1470  RT x( rhs.vec_ );
1471  if( x.nonZeros() == 0UL ) {
1472  reset( ~lhs );
1473  return;
1474  }
1475 
1476  // Evaluation of the left-hand side dense matrix operand
1477  LT A( rhs.mat_ );
1478 
1479  // Checking the evaluated operands
1480  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1481  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1482  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1483  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1484 
1485  // Performing the dense matrix-sparse vector multiplication
1486  smpAssign( ~lhs, A * x );
1487  }
1489  //**********************************************************************************************
1490 
1491  //**SMP assignment to sparse vectors************************************************************
1506  template< typename VT1 > // Type of the target sparse vector
1507  friend inline EnableIf_< UseSMPAssign<VT1> >
1509  {
1511 
1515 
1516  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1517 
1518  const ResultType tmp( rhs );
1519  smpAssign( ~lhs, tmp );
1520  }
1522  //**********************************************************************************************
1523 
1524  //**SMP addition assignment to dense vectors****************************************************
1539  template< typename VT1 > // Type of the target dense vector
1540  friend inline EnableIf_< UseSMPAssign<VT1> >
1542  {
1544 
1545  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1546 
1547  // Evaluation of the right-hand side sparse vector operand
1548  RT x( rhs.vec_ );
1549  if( x.nonZeros() == 0UL ) return;
1550 
1551  // Evaluation of the left-hand side dense matrix operand
1552  LT A( rhs.mat_ );
1553 
1554  // Checking the evaluated operands
1555  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1556  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1557  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1558  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1559 
1560  // Performing the dense matrix-sparse vector multiplication
1561  smpAddAssign( ~lhs, A * x );
1562  }
1564  //**********************************************************************************************
1565 
1566  //**SMP addition assignment to sparse vectors***************************************************
1567  // No special implementation for the SMP addition assignment to sparse vectors.
1568  //**********************************************************************************************
1569 
1570  //**SMP subtraction assignment to dense vectors*************************************************
1585  template< typename VT1 > // Type of the target dense vector
1586  friend inline EnableIf_< UseSMPAssign<VT1> >
1588  {
1590 
1591  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1592 
1593  // Evaluation of the right-hand side sparse vector operand
1594  RT x( rhs.vec_ );
1595  if( x.nonZeros() == 0UL ) return;
1596 
1597  // Evaluation of the left-hand side dense matrix operand
1598  LT A( rhs.mat_ );
1599 
1600  // Checking the evaluated operands
1601  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1602  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1603  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1604  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).size() , "Invalid vector size" );
1605 
1606  // Performing the dense matrix-sparse vector multiplication
1607  smpSubAssign( ~lhs, A * x );
1608  }
1610  //**********************************************************************************************
1611 
1612  //**SMP subtraction assignment to sparse vectors************************************************
1613  // No special implementation for the SMP subtraction assignment to sparse vectors.
1614  //**********************************************************************************************
1615 
1616  //**SMP multiplication assignment to dense vectors**********************************************
1631  template< typename VT1 > // Type of the target dense vector
1632  friend inline EnableIf_< UseSMPAssign<VT1> >
1634  {
1636 
1640 
1641  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1642 
1643  const ResultType tmp( rhs );
1644  smpMultAssign( ~lhs, tmp );
1645  }
1647  //**********************************************************************************************
1648 
1649  //**SMP multiplication assignment to sparse vectors*********************************************
1650  // No special implementation for the SMP multiplication assignment to sparse vectors.
1651  //**********************************************************************************************
1652 
1653  //**SMP division assignment to dense vectors****************************************************
1668  template< typename VT1 > // Type of the target dense vector
1669  friend inline EnableIf_< UseSMPAssign<VT1> >
1671  {
1673 
1677 
1678  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1679 
1680  const ResultType tmp( rhs );
1681  smpDivAssign( ~lhs, tmp );
1682  }
1684  //**********************************************************************************************
1685 
1686  //**SMP division assignment to sparse vectors***************************************************
1687  // No special implementation for the SMP division assignment to sparse vectors.
1688  //**********************************************************************************************
1689 
1690  //**Compile time checks*************************************************************************
1698  //**********************************************************************************************
1699 };
1700 //*************************************************************************************************
1701 
1702 
1703 
1704 
1705 //=================================================================================================
1706 //
1707 // GLOBAL BINARY ARITHMETIC OPERATORS
1708 //
1709 //=================================================================================================
1710 
1711 //*************************************************************************************************
1743 template< typename MT // Type of the left-hand side dense matrix
1744  , typename VT > // Type of the right-hand side sparse vector
1745 inline decltype(auto)
1746  operator*( const DenseMatrix<MT,true>& mat, const SparseVector<VT,false>& vec )
1747 {
1749 
1751 
1752  if( (~mat).columns() != (~vec).size() ) {
1753  BLAZE_THROW_INVALID_ARGUMENT( "Matrix and vector sizes do not match" );
1754  }
1755 
1756  using ReturnType = const TDMatSVecMultExpr<MT,VT>;
1757  return ReturnType( ~mat, ~vec );
1758 }
1759 //*************************************************************************************************
1760 
1761 
1762 
1763 
1764 //=================================================================================================
1765 //
1766 // SIZE SPECIALIZATIONS
1767 //
1768 //=================================================================================================
1769 
1770 //*************************************************************************************************
1772 template< typename MT, typename VT >
1773 struct Size< TDMatSVecMultExpr<MT,VT>, 0UL >
1774  : public Size<MT,0UL>
1775 {};
1777 //*************************************************************************************************
1778 
1779 
1780 
1781 
1782 //=================================================================================================
1783 //
1784 // ISALIGNED SPECIALIZATIONS
1785 //
1786 //=================================================================================================
1787 
1788 //*************************************************************************************************
1790 template< typename MT, typename VT >
1791 struct IsAligned< TDMatSVecMultExpr<MT,VT> >
1792  : public IsAligned<MT>
1793 {};
1795 //*************************************************************************************************
1796 
1797 } // namespace blaze
1798 
1799 #endif
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:329
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Header file for auxiliary alias declarations.
Header file for the blaze::checked and blaze::unchecked instances.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:71
RightOperand vec_
Right-hand side sparse vector of the multiplication expression.
Definition: TDMatSVecMultExpr.h:372
Header file for basic type definitions.
MultTrait_< MRT, VRT > ResultType
Result type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:196
If_< IsExpression< VT >, const VT, const VT &> RightOperand
Composite type of the right-hand side dense vector expression.
Definition: TDMatSVecMultExpr.h:207
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatSVecMultExpr.h:354
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a column dense or sparse vector type...
Definition: ColumnVector.h:61
EnableIf_< IsDenseMatrix< MT1 > > smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:164
Header file for the serial shim.
Header file for the IsDiagonal type trait.
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatSVecMultExpr.h:201
ResultType_< MT > MRT
Result type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:113
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the IsSame and IsStrictlySame type traits.
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:364
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:172
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:316
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:588
constexpr Unchecked unchecked
Global Unchecked instance.The blaze::unchecked instance is an optional token for the creation of view...
Definition: Check.h:138
EnableIf_< IsDenseVector< VT1 > > smpMultAssign(Vector< VT1, TF1 > &lhs, const Vector< VT2, TF2 > &rhs)
Default implementation of the SMP multiplication assignment of a vector to a dense vector...
Definition: DenseVector.h:193
Header file for the DenseVector base class.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:87
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:171
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:291
Header file for the Computation base class.
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:87
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Compile time check for data types.This type trait tests whether or not the given types can be combine...
Definition: IsSIMDCombinable.h:120
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:343
EnableIf_< IsDenseMatrix< MT1 > > smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:133
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:80
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
Constraint on the transpose flag of vector types.
ReturnType at(size_t index) const
Checked access to the vector elements.
Definition: TDMatSVecMultExpr.h:287
Compile time check for the alignment of data types.This type trait tests whether the given data type ...
Definition: IsAligned.h:87
Constraint on the data type.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:71
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
Header file for the DisableIf class template.
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:58
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:197
Header file for the If class template.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
Compile time check for data types with padding.This type trait tests whether the given data type empl...
Definition: IsPadded.h:76
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:102
LeftOperand mat_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatSVecMultExpr.h:371
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:3085
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
Base class for N-dimensional dense vectors.The DenseVector class is a base class for all arbitrarily ...
Definition: DenseVector.h:76
Header file for the IsLower type trait.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional vector type...
Definition: SparseVector.h:61
CompositeType_< VT > VCT
Composite type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:118
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:89
BLAZE_ALWAYS_INLINE size_t columns(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of columns of the matrix.
Definition: Matrix.h:506
#define BLAZE_CONSTRAINT_MUST_NOT_BE_MATMATMULTEXPR_TYPE(T)
Constraint on the data type.In case the given data type T is a matrix/matrix multiplication expressio...
Definition: MatMatMultExpr.h:87
LeftOperand leftOperand() const noexcept
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatSVecMultExpr.h:310
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATVECMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/vector ...
Definition: MatVecMultExpr.h:108
Constraint on the data type.
Header file for the exception macros of the math module.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:430
ReturnType operator[](size_t index) const
Subscript operator for the direct access to the vector elements.
Definition: TDMatSVecMultExpr.h:253
ResultType_< VT > VRT
Result type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:114
Constraint on the data type.
Header file for all forward declarations for expression class templates.
Constraint on the data type.
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
Header file for the IsPadded type trait.
RightOperand rightOperand() const noexcept
Returns the right-hand side sparse vector operand.
Definition: TDMatSVecMultExpr.h:320
Expression object for transpose dense matrix-sparse vector multiplications.The TDMatSVecMultExpr clas...
Definition: Forward.h:153
Compile time check for resizable data types.This type trait tests whether the given data type is a re...
Definition: IsResizable.h:75
Header file for the IsSIMDCombinable type trait.
Header file for the HasSIMDMult type trait.
Header file for run time assertion macros.
size_t size() const noexcept
Returns the current size/dimension of the vector.
Definition: TDMatSVecMultExpr.h:300
If_< IsExpression< MT >, const MT, const MT &> LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:204
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:154
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:131
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TDMatSVecMultExpr.h:332
ElementType_< MRT > MET
Element type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:115
EnableIf_< IsDenseVector< VT1 > > smpDivAssign(Vector< VT1, TF1 > &lhs, const Vector< VT2, TF2 > &rhs)
Default implementation of the SMP division assignment of a vector to a dense vector.
Definition: DenseVector.h:222
Header file for the reset shim.
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:296
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:94
Constraints on the storage order of matrix types.
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:816
TDMatSVecMultExpr(const MT &mat, const VT &vec) noexcept
Constructor for the TDMatSVecMultExpr class.
Definition: TDMatSVecMultExpr.h:239
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
CompositeType_< MT > MCT
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSVecMultExpr.h:117
Header file for the RemoveReference type trait.
ElementType_< VRT > VET
Element type of the right-hand side sparse vector expression.
Definition: TDMatSVecMultExpr.h:116
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:224
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
const Type & ReturnType
Return type for expression template evaluations.
Definition: CompressedMatrix.h:3080
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional vector type...
Definition: DenseVector.h:61
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: TDMatSVecMultExpr.h:199
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TDMatSVecMultExpr.h:344
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TDMatSVecMultExpr.h:364
Header file for the IsComputation type trait class.
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatSVecMultExpr.h:200
IfTrue_< evaluateVector, const VRT, CompositeType_< VT > > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: TDMatSVecMultExpr.h:213
Compile time evaluation of the size of vectors and matrices.The Size type trait evaluates the size of...
Definition: Size.h:80
Base class for sparse vectors.The SparseVector class is a base class for all arbitrarily sized (N-dim...
Definition: Forward.h:130
ElementType_< ResultType > ElementType
Resulting element type.
Definition: TDMatSVecMultExpr.h:198
Constraint on the data type.
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:423
Header file for the IsUpper type trait.
Header file for the MatVecMultExpr base class.
Constraint on the data type.
Header file for the IsResizable type trait.
Header file for the Size type trait.
Size type of the Blaze library.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
IfTrue_< evaluateMatrix, const MRT, MCT > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatSVecMultExpr.h:210
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the IsExpression type trait class.
Header file for the function trace functionality.