TSVecDMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TSVECDMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TSVECDMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
52 #include <blaze/math/Exception.h>
58 #include <blaze/math/shims/Reset.h>
60 #include <blaze/math/SIMD.h>
79 #include <blaze/util/Assert.h>
80 #include <blaze/util/DisableIf.h>
81 #include <blaze/util/EnableIf.h>
84 #include <blaze/util/mpl/If.h>
85 #include <blaze/util/Types.h>
88 
89 
90 namespace blaze {
91 
92 //=================================================================================================
93 //
94 // CLASS TSVECDMATMULTEXPR
95 //
96 //=================================================================================================
97 
98 //*************************************************************************************************
105 template< typename VT // Type of the left-hand side sparse vector
106  , typename MT > // Type of the right-hand side dense matrix
107 class TSVecDMatMultExpr
108  : public TVecMatMultExpr< DenseVector< TSVecDMatMultExpr<VT,MT>, true > >
109  , private Computation
110 {
111  private:
112  //**Type definitions****************************************************************************
119  //**********************************************************************************************
120 
121  //**********************************************************************************************
123  enum : bool { evaluateVector = IsComputation<VT>::value || RequiresEvaluation<VT>::value };
124  //**********************************************************************************************
125 
126  //**********************************************************************************************
128  enum : bool { evaluateMatrix = RequiresEvaluation<MT>::value };
129  //**********************************************************************************************
130 
131  //**********************************************************************************************
133 
137  template< typename T1 >
138  struct UseSMPAssign {
139  enum : bool { value = ( evaluateVector || evaluateMatrix ) };
140  };
142  //**********************************************************************************************
143 
144  //**********************************************************************************************
146 
150  template< typename T1, typename T2, typename T3 >
151  struct UseVectorizedKernel {
152  enum : bool { value = useOptimizedKernels &&
154  T1::simdEnabled && T3::simdEnabled &&
157  , ElementType_<T3> >::value &&
160  };
162  //**********************************************************************************************
163 
164  //**********************************************************************************************
166 
170  template< typename T1, typename T2, typename T3 >
171  struct UseOptimizedKernel {
172  enum : bool { value = useOptimizedKernels &&
173  !UseVectorizedKernel<T1,T2,T3>::value &&
175  !IsResizable< ElementType_<T1> >::value &&
177  };
179  //**********************************************************************************************
180 
181  //**********************************************************************************************
183 
186  template< typename T1, typename T2, typename T3 >
187  struct UseDefaultKernel {
188  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
189  !UseOptimizedKernel<T1,T2,T3>::value };
190  };
192  //**********************************************************************************************
193 
194  public:
195  //**Type definitions****************************************************************************
201  using ReturnType = const ElementType;
202  using CompositeType = const ResultType;
203 
205  using LeftOperand = If_< IsExpression<VT>, const VT, const VT& >;
206 
208  using RightOperand = If_< IsExpression<MT>, const MT, const MT& >;
209 
212 
215  //**********************************************************************************************
216 
217  //**Compilation flags***************************************************************************
219  enum : bool { simdEnabled = !IsDiagonal<MT>::value &&
220  MT::simdEnabled &&
223 
225  enum : bool { smpAssignable = !evaluateVector && VT::smpAssignable &&
226  !evaluateMatrix && MT::smpAssignable };
227  //**********************************************************************************************
228 
229  //**SIMD properties*****************************************************************************
231  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
232  //**********************************************************************************************
233 
234  //**Constructor*********************************************************************************
240  explicit inline TSVecDMatMultExpr( const VT& vec, const MT& mat ) noexcept
241  : vec_( vec ) // Left-hand side sparse vector of the multiplication expression
242  , mat_( mat ) // Right-hand side dense matrix of the multiplication expression
243  {
244  BLAZE_INTERNAL_ASSERT( vec_.size() == mat_.rows(), "Invalid vector and matrix sizes" );
245  }
246  //**********************************************************************************************
247 
248  //**Subscript operator**************************************************************************
254  inline ReturnType operator[]( size_t index ) const {
255  BLAZE_INTERNAL_ASSERT( index < mat_.columns(), "Invalid vector access index" );
256 
258  {
259  return vec_[index] * mat_(index,index);
260  }
261  else if( IsLower<MT>::value )
262  {
263  const size_t begin( IsStrictlyLower<MT>::value ? index+1UL : index );
264  const size_t n ( mat_.rows() - begin );
265  return subvector( vec_, begin, n ) * subvector( column( mat_, index ), begin, n );
266  }
267  else if( IsUpper<MT>::value )
268  {
269  const size_t n( IsStrictlyUpper<MT>::value ? index : index+1UL );
270  return subvector( vec_, 0UL, n ) * subvector( column( mat_, index ), 0UL, n );
271  }
272  else
273  {
274  return vec_ * column( mat_, index );
275  }
276  }
277  //**********************************************************************************************
278 
279  //**At function*********************************************************************************
286  inline ReturnType at( size_t index ) const {
287  if( index >= mat_.columns() ) {
288  BLAZE_THROW_OUT_OF_RANGE( "Invalid vector access index" );
289  }
290  return (*this)[index];
291  }
292  //**********************************************************************************************
293 
294  //**Size function*******************************************************************************
299  inline size_t size() const noexcept {
300  return mat_.columns();
301  }
302  //**********************************************************************************************
303 
304  //**Left operand access*************************************************************************
309  inline LeftOperand leftOperand() const noexcept {
310  return vec_;
311  }
312  //**********************************************************************************************
313 
314  //**Right operand access************************************************************************
319  inline RightOperand rightOperand() const noexcept {
320  return mat_;
321  }
322  //**********************************************************************************************
323 
324  //**********************************************************************************************
330  template< typename T >
331  inline bool canAlias( const T* alias ) const noexcept {
332  return vec_.isAliased( alias ) || mat_.isAliased( alias );
333  }
334  //**********************************************************************************************
335 
336  //**********************************************************************************************
342  template< typename T >
343  inline bool isAliased( const T* alias ) const noexcept {
344  return vec_.isAliased( alias ) || mat_.isAliased( alias );
345  }
346  //**********************************************************************************************
347 
348  //**********************************************************************************************
353  inline bool isAligned() const noexcept {
354  return mat_.isAligned();
355  }
356  //**********************************************************************************************
357 
358  //**********************************************************************************************
363  inline bool canSMPAssign() const noexcept {
364  return ( size() > SMP_TSVECDMATMULT_THRESHOLD );
365  }
366  //**********************************************************************************************
367 
368  private:
369  //**Member variables****************************************************************************
372  //**********************************************************************************************
373 
374  //**Assignment to dense vectors*****************************************************************
387  template< typename VT2 > // Type of the target dense vector
388  friend inline void assign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
389  {
391 
392  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
393 
394  // Evaluation of the left-hand side sparse vector operand
395  LT x( serial( rhs.vec_ ) );
396  if( x.nonZeros() == 0UL ) {
397  reset( ~lhs );
398  return;
399  }
400 
401  // Evaluation of the right-hand side dense matrix operand
402  RT A( serial( rhs.mat_ ) );
403 
404  // Checking the evaluated operands
405  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
406  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
407  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
408  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
409 
410  // Performing the sparse vector-dense matrix multiplication
411  TSVecDMatMultExpr::selectAssignKernel( ~lhs, x, A );
412  }
414  //**********************************************************************************************
415 
416  //**Default assignment to dense vectors*********************************************************
430  template< typename VT1 // Type of the left-hand side target vector
431  , typename VT2 // Type of the left-hand side vector operand
432  , typename MT1 > // Type of the right-hand side matrix operand
434  selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
435  {
437 
438  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
439 
440  const size_t N( A.columns() );
441 
442  ConstIterator element( x.begin() );
443  const ConstIterator end( x.end() );
444 
445  size_t last( 0UL );
446 
447  if( IsUpper<MT1>::value ) {
448  const size_t jend( IsStrictlyUpper<MT1>::value ? element->index()+1UL : element->index() );
449  for( size_t j=0UL; j<jend; ++j )
450  reset( y[j] );
451  }
452 
453  for( ; element!=end; ++element )
454  {
455  const size_t index( element->index() );
456 
458  {
459 
460 
461  for( size_t j=last; j<index; ++j )
462  reset( y[j] );
463 
464  y[index] = element->value() * A(index,index);
465  last = index + 1UL;
466  }
467  else
468  {
469  const size_t jbegin( ( IsUpper<MT1>::value )
470  ?( IsStrictlyUpper<MT1>::value ? index+1UL : index )
471  :( 0UL ) );
472  const size_t jend( ( IsLower<MT1>::value )
473  ?( IsStrictlyLower<MT1>::value ? index : index+1UL )
474  :( N ) );
475  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
476 
477  for( size_t j=jbegin; j<last; ++j ) {
478  y[j] += element->value() * A(index,j);
479  }
480  for( size_t j=last; j<jend; ++j ) {
481  y[j] = element->value() * A(index,j);
482  }
483 
484  last = jend;
485  }
486  }
487 
488  if( IsLower<MT1>::value ) {
489  for( size_t j=last; j<N; ++j )
490  reset( y[j] );
491  }
492  }
494  //**********************************************************************************************
495 
496  //**Optimized assignment to dense vectors*******************************************************
510  template< typename VT1 // Type of the left-hand side target vector
511  , typename VT2 // Type of the left-hand side vector operand
512  , typename MT1 > // Type of the right-hand side matrix operand
514  selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
515  {
517 
518  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
519 
520  const size_t N( A.columns() );
521 
522  ConstIterator element( x.begin() );
523  const ConstIterator end( x.end() );
524 
525  const size_t ipos( x.nonZeros() & size_t(-4) );
526  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
527 
528  if( ipos > 3UL )
529  {
530  const size_t i1( element->index() );
531  const VET v1( element->value() );
532  ++element;
533  const size_t i2( element->index() );
534  const VET v2( element->value() );
535  ++element;
536  const size_t i3( element->index() );
537  const VET v3( element->value() );
538  ++element;
539  const size_t i4( element->index() );
540  const VET v4( element->value() );
541  ++element;
542 
543  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
544 
545  for( size_t j=0UL; j<N; ++j ) {
546  y[j] = v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
547  }
548  }
549  else
550  {
551  const size_t i1( element->index() );
552  const VET v1( element->value() );
553  ++element;
554 
555  for( size_t j=0UL; j<N; ++j ) {
556  y[j] = v1 * A(i1,j);
557  }
558  }
559 
560  for( size_t i=(ipos>3UL)?(4UL):(1UL); (i+4UL)<=ipos; i+=4UL )
561  {
562  const size_t i1( element->index() );
563  const VET v1( element->value() );
564  ++element;
565  const size_t i2( element->index() );
566  const VET v2( element->value() );
567  ++element;
568  const size_t i3( element->index() );
569  const VET v3( element->value() );
570  ++element;
571  const size_t i4( element->index() );
572  const VET v4( element->value() );
573  ++element;
574 
575  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
576 
577  const size_t jbegin( ( IsUpper<MT1>::value )
578  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
579  :( 0UL ) );
580  const size_t jend( ( IsLower<MT1>::value )
581  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
582  :( N ) );
583  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
584 
585  for( size_t j=jbegin; j<jend; ++j ) {
586  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
587  }
588  }
589  for( ; element!=end; ++element )
590  {
591  const size_t i1( element->index() );
592  const VET v1( element->value() );
593 
594  const size_t jbegin( ( IsUpper<MT1>::value )
595  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
596  :( 0UL ) );
597  const size_t jend( ( IsLower<MT1>::value )
598  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
599  :( N ) );
600  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
601 
602  for( size_t j=jbegin; j<jend; ++j ) {
603  y[j] += v1 * A(i1,j);
604  }
605  }
606  }
608  //**********************************************************************************************
609 
610  //**Vectorized assignment to dense vectors******************************************************
624  template< typename VT1 // Type of the left-hand side target vector
625  , typename VT2 // Type of the left-hand side vector operand
626  , typename MT1 > // Type of the right-hand side matrix operand
628  selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
629  {
631 
632  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
633 
634  constexpr bool remainder( !IsPadded<VT1>::value || !IsPadded<MT1>::value );
635 
636  const size_t N( A.columns() );
637 
638  ConstIterator element( x.begin() );
639  const ConstIterator end( x.end() );
640 
641  const size_t ipos( x.nonZeros() & size_t(-4) );
642  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
643 
644  if( ipos > 3UL )
645  {
646  const size_t i1( element->index() );
647  const VET v1( element->value() );
648  ++element;
649  const size_t i2( element->index() );
650  const VET v2( element->value() );
651  ++element;
652  const size_t i3( element->index() );
653  const VET v3( element->value() );
654  ++element;
655  const size_t i4( element->index() );
656  const VET v4( element->value() );
657  ++element;
658 
659  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
660 
661  const SIMDType xmm1( set( v1 ) );
662  const SIMDType xmm2( set( v2 ) );
663  const SIMDType xmm3( set( v3 ) );
664  const SIMDType xmm4( set( v4 ) );
665 
666  const size_t jpos( remainder ? ( N & size_t(-SIMDSIZE) ) : N );
667  BLAZE_INTERNAL_ASSERT( !remainder || ( N - ( N % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
668 
669  size_t j( 0UL );
670 
671  for( ; j<jpos; j+=SIMDSIZE ) {
672  y.store( j, xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
673  }
674  for( ; remainder && j<N; ++j ) {
675  y[j] = v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
676  }
677  }
678  else
679  {
680  const size_t i1( element->index() );
681  const VET v1( element->value() );
682  ++element;
683 
684  const SIMDType xmm1( set( v1 ) );
685 
686  const size_t jpos( remainder ? ( N & size_t(-SIMDSIZE) ) : N );
687  BLAZE_INTERNAL_ASSERT( !remainder || ( N - ( N % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
688 
689  size_t j( 0UL );
690 
691  for( ; j<jpos; j+=SIMDSIZE ) {
692  y.store( j, xmm1 * A.load(i1,j) );
693  }
694  for( ; remainder && j<N; ++j ) {
695  y[j] = v1 * A(i1,j);
696  }
697  }
698 
699  for( size_t i=(ipos>3UL)?(4UL):(1UL); (i+4UL)<=ipos; i+=4UL )
700  {
701  const size_t i1( element->index() );
702  const VET v1( element->value() );
703  ++element;
704  const size_t i2( element->index() );
705  const VET v2( element->value() );
706  ++element;
707  const size_t i3( element->index() );
708  const VET v3( element->value() );
709  ++element;
710  const size_t i4( element->index() );
711  const VET v4( element->value() );
712  ++element;
713 
714  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
715 
716  const SIMDType xmm1( set( v1 ) );
717  const SIMDType xmm2( set( v2 ) );
718  const SIMDType xmm3( set( v3 ) );
719  const SIMDType xmm4( set( v4 ) );
720 
721  const size_t jbegin( ( IsUpper<MT1>::value )
722  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
723  :( 0UL ) );
724  const size_t jend( ( IsLower<MT1>::value )
725  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
726  :( N ) );
727  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
728 
729  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
730  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
731 
732  size_t j( jbegin );
733 
734  for( ; j<jpos; j+=SIMDSIZE ) {
735  y.store( j, y.load(j) + xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
736  }
737  for( ; remainder && j<jend; ++j ) {
738  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
739  }
740  }
741  for( ; element!=end; ++element )
742  {
743  const size_t i1( element->index() );
744  const VET v1( element->value() );
745 
746  const SIMDType xmm1( set( v1 ) );
747 
748  const size_t jbegin( ( IsUpper<MT1>::value )
749  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
750  :( 0UL ) );
751  const size_t jend( ( IsLower<MT1>::value )
752  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
753  :( N ) );
754  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
755 
756  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
757  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
758 
759  size_t j( jbegin );
760 
761  for( ; j<jpos; j+=SIMDSIZE ) {
762  y.store( j, y.load(j) + xmm1 * A.load(i1,j) );
763  }
764  for( ; remainder && j<jend; ++j ) {
765  y[j] += v1 * A(i1,j);
766  }
767  }
768  }
770  //**********************************************************************************************
771 
772  //**Assignment to sparse vectors****************************************************************
785  template< typename VT2 > // Type of the target sparse vector
786  friend inline void assign( SparseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
787  {
789 
793 
794  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
795 
796  const ResultType tmp( serial( rhs ) );
797  assign( ~lhs, tmp );
798  }
800  //**********************************************************************************************
801 
802  //**Addition assignment to dense vectors********************************************************
814  template< typename VT2 > // Type of the target dense vector
815  friend inline void addAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
816  {
818 
819  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
820 
821  // Evaluation of the left-hand side sparse vector operand
822  LT x( serial( rhs.vec_ ) );
823  if( x.nonZeros() == 0UL ) return;
824 
825  // Evaluation of the right-hand side dense matrix operand
826  RT A( serial( rhs.mat_ ) );
827 
828  // Checking the evaluated operands
829  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
830  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
831  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
832  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
833 
834  // Performing the sparse vector-dense matrix multiplication
835  TSVecDMatMultExpr::selectAddAssignKernel( ~lhs, x, A );
836  }
837  //**********************************************************************************************
838 
839  //**Default addition assignment to dense vectors************************************************
853  template< typename VT1 // Type of the left-hand side target vector
854  , typename VT2 // Type of the left-hand side vector operand
855  , typename MT1 > // Type of the right-hand side matrix operand
857  selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
858  {
860 
861  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
862 
863  const size_t N( A.columns() );
864 
865  ConstIterator element( x.begin() );
866  const ConstIterator end( x.end() );
867 
868  for( ; element!=end; ++element )
869  {
870  const size_t index( element->index() );
871 
873  {
874  y[index] += A(index,index) * element->value();
875  }
876  else
877  {
878  const size_t jbegin( ( IsUpper<MT1>::value )
879  ?( IsStrictlyUpper<MT1>::value ? index+1UL : index )
880  :( 0UL ) );
881  const size_t jend( ( IsLower<MT1>::value )
882  ?( IsStrictlyLower<MT1>::value ? index : index+1UL )
883  :( N ) );
884  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
885 
886  for( size_t j=jbegin; j<jend; ++j ) {
887  y[j] += element->value() * A(index,j);
888  }
889  }
890  }
891  }
893  //**********************************************************************************************
894 
895  //**Optimized addition assignment to dense vectors**********************************************
909  template< typename VT1 // Type of the left-hand side target vector
910  , typename VT2 // Type of the left-hand side vector operand
911  , typename MT1 > // Type of the right-hand side matrix operand
913  selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
914  {
916 
917  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
918 
919  const size_t N( A.columns() );
920 
921  ConstIterator element( x.begin() );
922  const ConstIterator end( x.end() );
923 
924  const size_t ipos( x.nonZeros() & size_t(-4) );
925  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
926 
927  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
928  {
929  const size_t i1( element->index() );
930  const VET v1( element->value() );
931  ++element;
932  const size_t i2( element->index() );
933  const VET v2( element->value() );
934  ++element;
935  const size_t i3( element->index() );
936  const VET v3( element->value() );
937  ++element;
938  const size_t i4( element->index() );
939  const VET v4( element->value() );
940  ++element;
941 
942  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
943 
944  const size_t jbegin( ( IsUpper<MT1>::value )
945  ?( IsStrictlyUpper<MT1>::value ? i+1UL : i1 )
946  :( 0UL ) );
947  const size_t jend( ( IsLower<MT1>::value )
948  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
949  :( N ) );
950  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
951 
952  for( size_t j=jbegin; j<jend; ++j ) {
953  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
954  }
955  }
956  for( ; element!=end; ++element )
957  {
958  const size_t i1( element->index() );
959  const VET v1( element->value() );
960 
961  const size_t jbegin( ( IsUpper<MT1>::value )
962  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
963  :( 0UL ) );
964  const size_t jend( ( IsLower<MT1>::value )
965  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
966  :( N ) );
967  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
968 
969  for( size_t j=jbegin; j<jend; ++j ) {
970  y[j] += v1 * A(i1,j);
971  }
972  }
973  }
975  //**********************************************************************************************
976 
977  //**Vectorized addition assignment to dense vectors*********************************************
991  template< typename VT1 // Type of the left-hand side target vector
992  , typename VT2 // Type of the left-hand side vector operand
993  , typename MT1 > // Type of the right-hand side matrix operand
995  selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
996  {
998 
999  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1000 
1001  constexpr bool remainder( !IsPadded<VT1>::value || !IsPadded<MT1>::value );
1002 
1003  const size_t N( A.columns() );
1004 
1005  ConstIterator element( x.begin() );
1006  const ConstIterator end( x.end() );
1007 
1008  const size_t ipos( x.nonZeros() & size_t(-4) );
1009  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1010 
1011  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1012  {
1013  const size_t i1( element->index() );
1014  const VET v1( element->value() );
1015  ++element;
1016  const size_t i2( element->index() );
1017  const VET v2( element->value() );
1018  ++element;
1019  const size_t i3( element->index() );
1020  const VET v3( element->value() );
1021  ++element;
1022  const size_t i4( element->index() );
1023  const VET v4( element->value() );
1024  ++element;
1025 
1026  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1027 
1028  const SIMDType xmm1( set( v1 ) );
1029  const SIMDType xmm2( set( v2 ) );
1030  const SIMDType xmm3( set( v3 ) );
1031  const SIMDType xmm4( set( v4 ) );
1032 
1033  const size_t jbegin( ( IsUpper<MT1>::value )
1034  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1035  :( 0UL ) );
1036  const size_t jend( ( IsLower<MT1>::value )
1037  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
1038  :( N ) );
1039  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1040 
1041  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1042  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1043 
1044  size_t j( jbegin );
1045 
1046  for( ; j<jpos; j+=SIMDSIZE ) {
1047  y.store( j, y.load(j) + xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
1048  }
1049  for( ; remainder && j<jend; ++j ) {
1050  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1051  }
1052  }
1053  for( ; element!=end; ++element )
1054  {
1055  const size_t i1( element->index() );
1056  const VET v1( element->value() );
1057 
1058  const SIMDType xmm1( set( v1 ) );
1059 
1060  const size_t jbegin( ( IsUpper<MT1>::value )
1061  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1062  :( 0UL ) );
1063  const size_t jend( ( IsLower<MT1>::value )
1064  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
1065  :( N ) );
1066  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1067 
1068  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1069  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1070 
1071  size_t j( jbegin );
1072 
1073  for( ; j<jpos; j+=SIMDSIZE ) {
1074  y.store( j, y.load(j) + xmm1 * A.load(i1,j) );
1075  }
1076  for( ; remainder && j<jend; ++j ) {
1077  y[j] += v1 * A(i1,j);
1078  }
1079  }
1080  }
1082  //**********************************************************************************************
1083 
1084  //**Addition assignment to sparse vectors*******************************************************
1085  // No special implementation for the addition assignment to sparse vectors.
1086  //**********************************************************************************************
1087 
1088  //**Subtraction assignment to dense vectors*****************************************************
1100  template< typename VT2 > // Type of the target dense vector
1101  friend inline void subAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1102  {
1104 
1105  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1106 
1107  // Evaluation of the left-hand side sparse vector operand
1108  LT x( serial( rhs.vec_ ) );
1109  if( x.nonZeros() == 0UL ) return;
1110 
1111  // Evaluation of the right-hand side dense matrix operand
1112  RT A( serial( rhs.mat_ ) );
1113 
1114  // Checking the evaluated operands
1115  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1116  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1117  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1118  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1119 
1120  // Performing the sparse vector-dense matrix multiplication
1121  TSVecDMatMultExpr::selectSubAssignKernel( ~lhs, x, A );
1122  }
1123  //**********************************************************************************************
1124 
1125  //**Default subtraction assignment to dense vectors*********************************************
1139  template< typename VT1 // Type of the left-hand side target vector
1140  , typename VT2 // Type of the left-hand side vector operand
1141  , typename MT1 > // Type of the right-hand side matrix operand
1143  selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1144  {
1146 
1147  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1148 
1149  const size_t N( A.columns() );
1150 
1151  ConstIterator element( x.begin() );
1152  const ConstIterator end( x.end() );
1153 
1154  for( ; element!=end; ++element )
1155  {
1156  const size_t index( element->index() );
1157 
1159  {
1160  y[index] -= A(index,index) * element->value();
1161  }
1162  else
1163  {
1164  const size_t jbegin( ( IsUpper<MT1>::value )
1165  ?( IsStrictlyUpper<MT1>::value ? index+1UL : index )
1166  :( 0UL ) );
1167  const size_t jend( ( IsLower<MT1>::value )
1168  ?( IsStrictlyLower<MT1>::value ? index : index+1UL )
1169  :( N ) );
1170  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1171 
1172  for( size_t j=jbegin; j<jend; ++j ) {
1173  y[j] -= element->value() * A(index,j);
1174  }
1175  }
1176  }
1177  }
1179  //**********************************************************************************************
1180 
1181  //**Optimized subtraction assignment to dense vectors*******************************************
1195  template< typename VT1 // Type of the left-hand side target vector
1196  , typename VT2 // Type of the left-hand side vector operand
1197  , typename MT1 > // Type of the right-hand side matrix operand
1199  selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1200  {
1202 
1203  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1204 
1205  const size_t N( A.columns() );
1206 
1207  ConstIterator element( x.begin() );
1208  const ConstIterator end( x.end() );
1209 
1210  const size_t ipos( x.nonZeros() & size_t(-4) );
1211  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1212 
1213  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1214  {
1215  const size_t i1( element->index() );
1216  const VET v1( element->value() );
1217  ++element;
1218  const size_t i2( element->index() );
1219  const VET v2( element->value() );
1220  ++element;
1221  const size_t i3( element->index() );
1222  const VET v3( element->value() );
1223  ++element;
1224  const size_t i4( element->index() );
1225  const VET v4( element->value() );
1226  ++element;
1227 
1228  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1229 
1230  const size_t jbegin( ( IsUpper<MT1>::value )
1231  ?( IsStrictlyUpper<MT1>::value ? i+1UL : i1 )
1232  :( 0UL ) );
1233  const size_t jend( ( IsLower<MT1>::value )
1234  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
1235  :( N ) );
1236  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1237 
1238  for( size_t j=jbegin; j<jend; ++j ) {
1239  y[j] -= v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1240  }
1241  }
1242  for( ; element!=end; ++element )
1243  {
1244  const size_t i1( element->index() );
1245  const VET v1( element->value() );
1246 
1247  const size_t jbegin( ( IsUpper<MT1>::value )
1248  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
1249  :( 0UL ) );
1250  const size_t jend( ( IsLower<MT1>::value )
1251  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
1252  :( N ) );
1253  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1254 
1255  for( size_t j=jbegin; j<jend; ++j ) {
1256  y[j] -= v1 * A(i1,j);
1257  }
1258  }
1259  }
1261  //**********************************************************************************************
1262 
1263  //**Vectorized subtraction assignment to dense vectors******************************************
1277  template< typename VT1 // Type of the left-hand side target vector
1278  , typename VT2 // Type of the left-hand side vector operand
1279  , typename MT1 > // Type of the right-hand side matrix operand
1281  selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1282  {
1284 
1285  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1286 
1287  constexpr bool remainder( !IsPadded<VT1>::value || !IsPadded<MT1>::value );
1288 
1289  const size_t N( A.columns() );
1290 
1291  ConstIterator element( x.begin() );
1292  const ConstIterator end( x.end() );
1293 
1294  const size_t ipos( x.nonZeros() & size_t(-4) );
1295  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1296 
1297  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1298  {
1299  const size_t i1( element->index() );
1300  const VET v1( element->value() );
1301  ++element;
1302  const size_t i2( element->index() );
1303  const VET v2( element->value() );
1304  ++element;
1305  const size_t i3( element->index() );
1306  const VET v3( element->value() );
1307  ++element;
1308  const size_t i4( element->index() );
1309  const VET v4( element->value() );
1310  ++element;
1311 
1312  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1313 
1314  const SIMDType xmm1( set( v1 ) );
1315  const SIMDType xmm2( set( v2 ) );
1316  const SIMDType xmm3( set( v3 ) );
1317  const SIMDType xmm4( set( v4 ) );
1318 
1319  const size_t jbegin( ( IsUpper<MT1>::value )
1320  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1321  :( 0UL ) );
1322  const size_t jend( ( IsLower<MT1>::value )
1323  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
1324  :( N ) );
1325  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1326 
1327  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1328  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1329 
1330  size_t j( jbegin );
1331 
1332  for( ; j<jpos; j+=SIMDSIZE ) {
1333  y.store( j, y.load(j) - xmm1 * A.load(i1,j) - xmm2 * A.load(i2,j) - xmm3 * A.load(i3,j) - xmm4 * A.load(i4,j) );
1334  }
1335  for( ; remainder && j<jend; ++j ) {
1336  y[j] -= v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1337  }
1338  }
1339  for( ; element!=end; ++element )
1340  {
1341  const size_t i1( element->index() );
1342  const VET v1( element->value() );
1343 
1344  const SIMDType xmm1( set( v1 ) );
1345 
1346  const size_t jbegin( ( IsUpper<MT1>::value )
1347  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1348  :( 0UL ) );
1349  const size_t jend( ( IsLower<MT1>::value )
1350  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
1351  :( N ) );
1352  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1353 
1354  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1355  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1356 
1357  size_t j( jbegin );
1358 
1359  for( ; j<jpos; j+=SIMDSIZE ) {
1360  y.store( j, y.load(j) - xmm1 * A.load(i1,j) );
1361  }
1362  for( ; remainder && j<jend; ++j ) {
1363  y[j] -= v1 * A(i1,j);
1364  }
1365  }
1366  }
1368  //**********************************************************************************************
1369 
1370  //**Subtraction assignment to sparse vectors****************************************************
1371  // No special implementation for the subtraction assignment to sparse vectors.
1372  //**********************************************************************************************
1373 
1374  //**Multiplication assignment to dense vectors**************************************************
1386  template< typename VT2 > // Type of the target dense vector
1387  friend inline void multAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1388  {
1390 
1394 
1395  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1396 
1397  const ResultType tmp( serial( rhs ) );
1398  multAssign( ~lhs, tmp );
1399  }
1400  //**********************************************************************************************
1401 
1402  //**Multiplication assignment to sparse vectors*************************************************
1403  // No special implementation for the multiplication assignment to sparse vectors.
1404  //**********************************************************************************************
1405 
1406  //**Division assignment to dense vectors********************************************************
1418  template< typename VT2 > // Type of the target dense vector
1419  friend inline void divAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1420  {
1422 
1426 
1427  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1428 
1429  const ResultType tmp( serial( rhs ) );
1430  divAssign( ~lhs, tmp );
1431  }
1432  //**********************************************************************************************
1433 
1434  //**Division assignment to sparse vectors*******************************************************
1435  // No special implementation for the division assignment to sparse vectors.
1436  //**********************************************************************************************
1437 
1438  //**SMP assignment to dense vectors*************************************************************
1453  template< typename VT2 > // Type of the target dense vector
1454  friend inline EnableIf_< UseSMPAssign<VT2> >
1456  {
1458 
1459  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1460 
1461  // Evaluation of the left-hand side sparse vector operand
1462  LT x( rhs.vec_ );
1463  if( x.nonZeros() == 0UL ) {
1464  reset( ~lhs );
1465  return;
1466  }
1467 
1468  // Evaluation of the right-hand side dense matrix operand
1469  RT A( rhs.mat_ );
1470 
1471  // Checking the evaluated operands
1472  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1473  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1474  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1475  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1476 
1477  // Performing the sparse vector-dense matrix multiplication
1478  smpAssign( ~lhs, x * A );
1479  }
1481  //**********************************************************************************************
1482 
1483  //**SMP assignment to sparse vectors************************************************************
1498  template< typename VT2 > // Type of the target sparse vector
1499  friend inline EnableIf_< UseSMPAssign<VT2> >
1501  {
1503 
1507 
1508  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1509 
1510  const ResultType tmp( rhs );
1511  smpAssign( ~lhs, tmp );
1512  }
1514  //**********************************************************************************************
1515 
1516  //**SMP addition assignment to dense vectors****************************************************
1530  template< typename VT2 > // Type of the target dense vector
1531  friend inline EnableIf_< UseSMPAssign<VT2> >
1533  {
1535 
1536  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1537 
1538  // Evaluation of the left-hand side sparse vector operand
1539  LT x( rhs.vec_ );
1540  if( x.nonZeros() == 0UL ) return;
1541 
1542  // Evaluation of the right-hand side dense matrix operand
1543  RT A( rhs.mat_ );
1544 
1545  // Checking the evaluated operands
1546  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1547  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1548  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1549  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1550 
1551  // Performing the sparse vector-dense matrix multiplication
1552  smpAddAssign( ~lhs, x * A );
1553  }
1554  //**********************************************************************************************
1555 
1556  //**SMP addition assignment to sparse vectors***************************************************
1557  // No special implementation for the SMP addition assignment to sparse vectors.
1558  //**********************************************************************************************
1559 
1560  //**SMP subtraction assignment to dense vectors*************************************************
1574  template< typename VT2 > // Type of the target dense vector
1575  friend inline EnableIf_< UseSMPAssign<VT2> >
1577  {
1579 
1580  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1581 
1582  // Evaluation of the left-hand side sparse vector operand
1583  LT x( rhs.vec_ );
1584  if( x.nonZeros() == 0UL ) return;
1585 
1586  // Evaluation of the right-hand side dense matrix operand
1587  RT A( rhs.mat_ );
1588 
1589  // Checking the evaluated operands
1590  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1591  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1592  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1593  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1594 
1595  // Performing the sparse vector-dense matrix multiplication
1596  smpSubAssign( ~lhs, x * A );
1597  }
1598  //**********************************************************************************************
1599 
1600  //**SMP subtraction assignment to sparse vectors************************************************
1601  // No special implementation for the SMP subtraction assignment to sparse vectors.
1602  //**********************************************************************************************
1603 
1604  //**SMP multiplication assignment to dense vectors**********************************************
1618  template< typename VT2 > // Type of the target dense vector
1619  friend inline EnableIf_< UseSMPAssign<VT2> >
1621  {
1623 
1627 
1628  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1629 
1630  const ResultType tmp( rhs );
1631  smpMultAssign( ~lhs, tmp );
1632  }
1633  //**********************************************************************************************
1634 
1635  //**SMP multiplication assignment to sparse vectors*********************************************
1636  // No special implementation for the SMP multiplication assignment to sparse vectors.
1637  //**********************************************************************************************
1638 
1639  //**SMP division assignment to dense vectors****************************************************
1653  template< typename VT2 > // Type of the target dense vector
1654  friend inline EnableIf_< UseSMPAssign<VT2> >
1656  {
1658 
1662 
1663  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1664 
1665  const ResultType tmp( rhs );
1666  smpDivAssign( ~lhs, tmp );
1667  }
1668  //**********************************************************************************************
1669 
1670  //**SMP division assignment to sparse vectors***************************************************
1671  // No special implementation for the SMP division assignment to sparse vectors.
1672  //**********************************************************************************************
1673 
1674  //**Compile time checks*************************************************************************
1682  //**********************************************************************************************
1683 };
1684 //*************************************************************************************************
1685 
1686 
1687 
1688 
1689 //=================================================================================================
1690 //
1691 // GLOBAL BINARY ARITHMETIC OPERATORS
1692 //
1693 //=================================================================================================
1694 
1695 //*************************************************************************************************
1726 template< typename VT // Type of the left-hand side sparse vector
1727  , typename MT > // Type of the right-hand side dense matrix
1728 inline decltype(auto)
1729  operator*( const SparseVector<VT,true>& vec, const DenseMatrix<MT,false>& mat )
1730 {
1732 
1734 
1735  if( (~vec).size() != (~mat).rows() ) {
1736  BLAZE_THROW_INVALID_ARGUMENT( "Vector and matrix sizes do not match" );
1737  }
1738 
1739  using ReturnType = const TSVecDMatMultExpr<VT,MT>;
1740  return ReturnType( ~vec, ~mat );
1741 }
1742 //*************************************************************************************************
1743 
1744 
1745 
1746 
1747 //=================================================================================================
1748 //
1749 // GLOBAL RESTRUCTURING BINARY ARITHMETIC OPERATORS
1750 //
1751 //=================================================================================================
1752 
1753 //*************************************************************************************************
1767 template< typename VT // Type of the left-hand side sparse vector
1768  , typename MT > // Matrix base type of the right-hand side expression
1769 inline decltype(auto)
1770  operator*( const SparseVector<VT,true>& vec, const MatMatMultExpr<MT>& mat )
1771 {
1773 
1774  return ( vec * (~mat).leftOperand() ) * (~mat).rightOperand();
1775 }
1777 //*************************************************************************************************
1778 
1779 
1780 
1781 
1782 //=================================================================================================
1783 //
1784 // SIZE SPECIALIZATIONS
1785 //
1786 //=================================================================================================
1787 
1788 //*************************************************************************************************
1790 template< typename VT, typename MT >
1791 struct Size< TSVecDMatMultExpr<VT,MT> >
1792  : public Columns<MT>
1793 {};
1795 //*************************************************************************************************
1796 
1797 
1798 
1799 
1800 //=================================================================================================
1801 //
1802 // ISALIGNED SPECIALIZATIONS
1803 //
1804 //=================================================================================================
1805 
1806 //*************************************************************************************************
1808 template< typename VT, typename MT >
1809 struct IsAligned< TSVecDMatMultExpr<VT,MT> >
1810  : public BoolConstant< IsAligned<MT>::value >
1811 {};
1813 //*************************************************************************************************
1814 
1815 } // namespace blaze
1816 
1817 #endif
friend EnableIf_< UseSMPAssign< VT2 > > smpDivAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP division assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ...
Definition: TSVecDMatMultExpr.h:1655
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
friend EnableIf_< UseSMPAssign< VT2 > > smpAddAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP addition assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ...
Definition: TSVecDMatMultExpr.h:1532
Header file for auxiliary alias declarations.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:72
size_t size() const noexcept
Returns the current size/dimension of the vector.
Definition: TSVecDMatMultExpr.h:299
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TSVecDMatMultExpr.h:343
friend void multAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Multiplication assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ...
Definition: TSVecDMatMultExpr.h:1387
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:201
Header file for basic type definitions.
Expression object for transpose sparse vector-dense matrix multiplications.The TSVecDMatMultExpr clas...
Definition: Forward.h:173
Subvector< VT, AF > subvector(Vector< VT, TF > &vector, size_t index, size_t size)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:322
friend EnableIf_< UseSMPAssign< VT2 > > smpMultAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP multiplication assignment of a transpose sparse vector-dense matrix multiplication to a dense vec...
Definition: TSVecDMatMultExpr.h:1620
Header file for the serial shim.
Header file for the IsDiagonal type trait.
Generic wrapper for a compile time constant integral value.The IntegralConstant class template repres...
Definition: IntegralConstant.h:71
CompositeType_< MT > MCT
Composite type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:118
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the IsSame and IsStrictlySame type traits.
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:198
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:172
LeftOperand vec_
Left-hand side sparse vector of the multiplication expression.
Definition: TSVecDMatMultExpr.h:370
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:316
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:560
ElementType_< VRT > VET
Element type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:115
ElementType_< ResultType > ElementType
Resulting element type.
Definition: TSVecDMatMultExpr.h:199
IfTrue_< evaluateVector, const VRT, VCT > LT
Type for the assignment of the left-hand side sparse vector operand.
Definition: TSVecDMatMultExpr.h:211
Header file for the DenseVector base class.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:88
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:171
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TSVecDMatMultExpr.h:331
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:250
Column< MT > column(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific column of the given matrix.
Definition: Column.h:124
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:88
Constraints on the storage order of matrix types.
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:198
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Compile time check for data types.This type trait tests whether or not the given types can be combine...
Definition: IsSIMDCombinable.h:120
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:343
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:78
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
Compile time check for the alignment of data types.This type trait tests whether the given data type ...
Definition: IsAligned.h:87
Constraint on the data type.
RightOperand mat_
Right-hand side dense matrix of the multiplication expression.
Definition: TSVecDMatMultExpr.h:371
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:72
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
MultTrait_< VRT, MRT > ResultType
Result type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:197
Header file for the DisableIf class template.
friend void addAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Addition assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:815
Header file for the multiplication trait.
RightOperand rightOperand() const noexcept
Returns the right-hand side dense matrix operand.
Definition: TSVecDMatMultExpr.h:319
Header file for the IsStrictlyUpper type trait.
LeftOperand leftOperand() const noexcept
Returns the left-hand side sparse vector operand.
Definition: TSVecDMatMultExpr.h:309
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
Header file for the If class template.
friend void divAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Division assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:1419
Compile time check for data types with padding.This type trait tests whether the given data type empl...
Definition: IsPadded.h:76
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:102
const ResultType CompositeType
Data type for composite expression templates.
Definition: TSVecDMatMultExpr.h:202
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
Header file for the Columns type trait.
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:3087
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
Base class for N-dimensional dense vectors.The DenseVector class is a base class for all arbitrarily ...
Definition: DenseVector.h:70
ResultType_< MT > MRT
Result type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:114
Header file for the IsLower type trait.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional vector type...
Definition: SparseVector.h:61
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:90
If_< IsExpression< VT >, const VT, const VT &> LeftOperand
Composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:205
#define BLAZE_CONSTRAINT_MUST_NOT_BE_MATMATMULTEXPR_TYPE(T)
Constraint on the data type.In case the given data type T is a matrix/matrix multiplication expressio...
Definition: MatMatMultExpr.h:88
ElementType_< MRT > MET
Element type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:116
Constraint on the data type.
Header file for the exception macros of the math module.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:264
decltype(auto) operator*(const DenseMatrix< MT1, false > &lhs, const DenseMatrix< MT2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:8893
Constraint on the data type.
Header file for all forward declarations for expression class templates.
Constraint on the data type.
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
IfTrue_< evaluateMatrix, const MRT, MCT > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: TSVecDMatMultExpr.h:214
Compile time check for resizable data types.This type trait tests whether the given data type is a re...
Definition: IsResizable.h:75
ResultType_< VT > VRT
Result type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:113
Base class for all matrix/matrix multiplication expression templates.The MatMatMultExpr class serves ...
Definition: MatMatMultExpr.h:67
Header file for the IsSIMDCombinable type trait.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TSVecDMatMultExpr.h:353
Header file for run time assertion macros.
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:154
Header file for the reset shim.
Constraint on the data type.
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:296
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:94
ReturnType operator[](size_t index) const
Subscript operator for the direct access to the vector elements.
Definition: TSVecDMatMultExpr.h:254
Header file for the TVecMatMultExpr base class.
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:819
TSVecDMatMultExpr(const VT &vec, const MT &mat) noexcept
Constructor for the TSVecDMatMultExpr class.
Definition: TSVecDMatMultExpr.h:240
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: TSVecDMatMultExpr.h:200
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TSVecDMatMultExpr.h:363
Header file for the RemoveReference type trait.
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:224
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
BLAZE_ALWAYS_INLINE size_t rows(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of rows of the matrix.
Definition: Matrix.h:324
const Type & ReturnType
Return type for expression template evaluations.
Definition: CompressedMatrix.h:3082
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional vector type...
Definition: DenseVector.h:61
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_TVECMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid vector/matrix ...
Definition: TVecMatMultExpr.h:109
Header file for the IsComputation type trait class.
Compile time evaluation of the size of a vector.The Size type trait evaluates the size of the given v...
Definition: Size.h:74
Base class for sparse vectors.The SparseVector class is a base class for all arbitrarily sized (N-dim...
Definition: Forward.h:130
Header file for the IntegralConstant class template.
Compile time evaluation of the number of columns of a matrix.The Columns type trait evaluates the num...
Definition: Columns.h:75
#define BLAZE_CONSTRAINT_MUST_BE_ROW_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a row dense or sparse vector type (i...
Definition: RowVector.h:61
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:423
friend void subAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Subtraction assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:1101
Header file for the IsUpper type trait.
Constraint on the data type.
ReturnType at(size_t index) const
Checked access to the vector elements.
Definition: TSVecDMatMultExpr.h:286
friend EnableIf_< UseSMPAssign< VT2 > > smpSubAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP subtraction assignment of a transpose sparse vector-dense matrix multiplication to a dense vector...
Definition: TSVecDMatMultExpr.h:1576
Header file for the IsResizable type trait.
Header file for the Size type trait.
Size type of the Blaze library.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
If_< IsExpression< MT >, const MT, const MT &> RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TSVecDMatMultExpr.h:208
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
CompositeType_< VT > VCT
Composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:117
Constraint on the transpose flag of vector types.
Header file for the IsExpression type trait class.
Header file for the function trace functionality.