TSVecDMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_TSVECDMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_TSVECDMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
52 #include <blaze/math/Exception.h>
58 #include <blaze/math/shims/Reset.h>
60 #include <blaze/math/SIMD.h>
77 #include <blaze/math/views/Check.h>
80 #include <blaze/util/Assert.h>
81 #include <blaze/util/DisableIf.h>
82 #include <blaze/util/EnableIf.h>
84 #include <blaze/util/mpl/If.h>
85 #include <blaze/util/Types.h>
88 
89 
90 namespace blaze {
91 
92 //=================================================================================================
93 //
94 // CLASS TSVECDMATMULTEXPR
95 //
96 //=================================================================================================
97 
98 //*************************************************************************************************
105 template< typename VT // Type of the left-hand side sparse vector
106  , typename MT > // Type of the right-hand side dense matrix
107 class TSVecDMatMultExpr
108  : public TVecMatMultExpr< DenseVector< TSVecDMatMultExpr<VT,MT>, true > >
109  , private Computation
110 {
111  private:
112  //**Type definitions****************************************************************************
119  //**********************************************************************************************
120 
121  //**********************************************************************************************
123  enum : bool { evaluateVector = IsComputation<VT>::value || RequiresEvaluation<VT>::value };
124  //**********************************************************************************************
125 
126  //**********************************************************************************************
128  enum : bool { evaluateMatrix = RequiresEvaluation<MT>::value };
129  //**********************************************************************************************
130 
131  //**********************************************************************************************
133 
137  template< typename T1 >
138  struct UseSMPAssign {
139  enum : bool { value = ( evaluateVector || evaluateMatrix ) };
140  };
142  //**********************************************************************************************
143 
144  //**********************************************************************************************
146 
150  template< typename T1, typename T2, typename T3 >
151  struct UseVectorizedKernel {
152  enum : bool { value = useOptimizedKernels &&
154  T1::simdEnabled && T3::simdEnabled &&
157  , ElementType_<T3> >::value &&
160  };
162  //**********************************************************************************************
163 
164  //**********************************************************************************************
166 
170  template< typename T1, typename T2, typename T3 >
171  struct UseOptimizedKernel {
172  enum : bool { value = useOptimizedKernels &&
173  !UseVectorizedKernel<T1,T2,T3>::value &&
175  !IsResizable< ElementType_<T1> >::value &&
177  };
179  //**********************************************************************************************
180 
181  //**********************************************************************************************
183 
186  template< typename T1, typename T2, typename T3 >
187  struct UseDefaultKernel {
188  enum : bool { value = !UseVectorizedKernel<T1,T2,T3>::value &&
189  !UseOptimizedKernel<T1,T2,T3>::value };
190  };
192  //**********************************************************************************************
193 
194  public:
195  //**Type definitions****************************************************************************
201  using ReturnType = const ElementType;
202  using CompositeType = const ResultType;
203 
205  using LeftOperand = If_< IsExpression<VT>, const VT, const VT& >;
206 
208  using RightOperand = If_< IsExpression<MT>, const MT, const MT& >;
209 
212 
215  //**********************************************************************************************
216 
217  //**Compilation flags***************************************************************************
219  enum : bool { simdEnabled = !IsDiagonal<MT>::value &&
220  MT::simdEnabled &&
223 
225  enum : bool { smpAssignable = !evaluateVector && VT::smpAssignable &&
226  !evaluateMatrix && MT::smpAssignable };
227  //**********************************************************************************************
228 
229  //**SIMD properties*****************************************************************************
231  enum : size_t { SIMDSIZE = SIMDTrait<ElementType>::size };
232  //**********************************************************************************************
233 
234  //**Constructor*********************************************************************************
240  explicit inline TSVecDMatMultExpr( const VT& vec, const MT& mat ) noexcept
241  : vec_( vec ) // Left-hand side sparse vector of the multiplication expression
242  , mat_( mat ) // Right-hand side dense matrix of the multiplication expression
243  {
244  BLAZE_INTERNAL_ASSERT( vec_.size() == mat_.rows(), "Invalid vector and matrix sizes" );
245  }
246  //**********************************************************************************************
247 
248  //**Subscript operator**************************************************************************
254  inline ReturnType operator[]( size_t index ) const {
255  BLAZE_INTERNAL_ASSERT( index < mat_.columns(), "Invalid vector access index" );
256 
258  {
259  return vec_[index] * mat_(index,index);
260  }
261  else if( IsLower<MT>::value )
262  {
263  const size_t begin( IsStrictlyLower<MT>::value ? index+1UL : index );
264  const size_t n ( mat_.rows() - begin );
265  return subvector( vec_, begin, n, unchecked ) *
266  subvector( column( mat_, index, unchecked ), begin, n, unchecked );
267  }
268  else if( IsUpper<MT>::value )
269  {
270  const size_t n( IsStrictlyUpper<MT>::value ? index : index+1UL );
271  return subvector( vec_, 0UL, n, unchecked ) *
272  subvector( column( mat_, index, unchecked ), 0UL, n, unchecked );
273  }
274  else
275  {
276  return vec_ * column( mat_, index, unchecked );
277  }
278  }
279  //**********************************************************************************************
280 
281  //**At function*********************************************************************************
288  inline ReturnType at( size_t index ) const {
289  if( index >= mat_.columns() ) {
290  BLAZE_THROW_OUT_OF_RANGE( "Invalid vector access index" );
291  }
292  return (*this)[index];
293  }
294  //**********************************************************************************************
295 
296  //**Size function*******************************************************************************
301  inline size_t size() const noexcept {
302  return mat_.columns();
303  }
304  //**********************************************************************************************
305 
306  //**Left operand access*************************************************************************
311  inline LeftOperand leftOperand() const noexcept {
312  return vec_;
313  }
314  //**********************************************************************************************
315 
316  //**Right operand access************************************************************************
321  inline RightOperand rightOperand() const noexcept {
322  return mat_;
323  }
324  //**********************************************************************************************
325 
326  //**********************************************************************************************
332  template< typename T >
333  inline bool canAlias( const T* alias ) const noexcept {
334  return vec_.isAliased( alias ) || mat_.isAliased( alias );
335  }
336  //**********************************************************************************************
337 
338  //**********************************************************************************************
344  template< typename T >
345  inline bool isAliased( const T* alias ) const noexcept {
346  return vec_.isAliased( alias ) || mat_.isAliased( alias );
347  }
348  //**********************************************************************************************
349 
350  //**********************************************************************************************
355  inline bool isAligned() const noexcept {
356  return mat_.isAligned();
357  }
358  //**********************************************************************************************
359 
360  //**********************************************************************************************
365  inline bool canSMPAssign() const noexcept {
366  return ( size() > SMP_TSVECDMATMULT_THRESHOLD );
367  }
368  //**********************************************************************************************
369 
370  private:
371  //**Member variables****************************************************************************
374  //**********************************************************************************************
375 
376  //**Assignment to dense vectors*****************************************************************
389  template< typename VT2 > // Type of the target dense vector
390  friend inline void assign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
391  {
393 
394  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
395 
396  // Evaluation of the left-hand side sparse vector operand
397  LT x( serial( rhs.vec_ ) );
398  if( x.nonZeros() == 0UL ) {
399  reset( ~lhs );
400  return;
401  }
402 
403  // Evaluation of the right-hand side dense matrix operand
404  RT A( serial( rhs.mat_ ) );
405 
406  // Checking the evaluated operands
407  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
408  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
409  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
410  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
411 
412  // Performing the sparse vector-dense matrix multiplication
413  TSVecDMatMultExpr::selectAssignKernel( ~lhs, x, A );
414  }
416  //**********************************************************************************************
417 
418  //**Default assignment to dense vectors*********************************************************
432  template< typename VT1 // Type of the left-hand side target vector
433  , typename VT2 // Type of the left-hand side vector operand
434  , typename MT1 > // Type of the right-hand side matrix operand
436  selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
437  {
439 
440  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
441 
442  const size_t N( A.columns() );
443 
444  ConstIterator element( x.begin() );
445  const ConstIterator end( x.end() );
446 
447  size_t last( 0UL );
448 
449  if( IsUpper<MT1>::value ) {
450  const size_t jend( IsStrictlyUpper<MT1>::value ? element->index()+1UL : element->index() );
451  for( size_t j=0UL; j<jend; ++j )
452  reset( y[j] );
453  }
454 
455  for( ; element!=end; ++element )
456  {
457  const size_t index( element->index() );
458 
460  {
461 
462 
463  for( size_t j=last; j<index; ++j )
464  reset( y[j] );
465 
466  y[index] = element->value() * A(index,index);
467  last = index + 1UL;
468  }
469  else
470  {
471  const size_t jbegin( ( IsUpper<MT1>::value )
472  ?( IsStrictlyUpper<MT1>::value ? index+1UL : index )
473  :( 0UL ) );
474  const size_t jend( ( IsLower<MT1>::value )
475  ?( IsStrictlyLower<MT1>::value ? index : index+1UL )
476  :( N ) );
477  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
478 
479  for( size_t j=jbegin; j<last; ++j ) {
480  y[j] += element->value() * A(index,j);
481  }
482  for( size_t j=last; j<jend; ++j ) {
483  y[j] = element->value() * A(index,j);
484  }
485 
486  last = jend;
487  }
488  }
489 
490  if( IsLower<MT1>::value ) {
491  for( size_t j=last; j<N; ++j )
492  reset( y[j] );
493  }
494  }
496  //**********************************************************************************************
497 
498  //**Optimized assignment to dense vectors*******************************************************
512  template< typename VT1 // Type of the left-hand side target vector
513  , typename VT2 // Type of the left-hand side vector operand
514  , typename MT1 > // Type of the right-hand side matrix operand
516  selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
517  {
519 
520  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
521 
522  const size_t N( A.columns() );
523 
524  ConstIterator element( x.begin() );
525  const ConstIterator end( x.end() );
526 
527  const size_t ipos( x.nonZeros() & size_t(-4) );
528  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
529 
530  if( ipos > 3UL )
531  {
532  const size_t i1( element->index() );
533  const VET v1( element->value() );
534  ++element;
535  const size_t i2( element->index() );
536  const VET v2( element->value() );
537  ++element;
538  const size_t i3( element->index() );
539  const VET v3( element->value() );
540  ++element;
541  const size_t i4( element->index() );
542  const VET v4( element->value() );
543  ++element;
544 
545  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
546 
547  for( size_t j=0UL; j<N; ++j ) {
548  y[j] = v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
549  }
550  }
551  else
552  {
553  const size_t i1( element->index() );
554  const VET v1( element->value() );
555  ++element;
556 
557  for( size_t j=0UL; j<N; ++j ) {
558  y[j] = v1 * A(i1,j);
559  }
560  }
561 
562  for( size_t i=(ipos>3UL)?(4UL):(1UL); (i+4UL)<=ipos; i+=4UL )
563  {
564  const size_t i1( element->index() );
565  const VET v1( element->value() );
566  ++element;
567  const size_t i2( element->index() );
568  const VET v2( element->value() );
569  ++element;
570  const size_t i3( element->index() );
571  const VET v3( element->value() );
572  ++element;
573  const size_t i4( element->index() );
574  const VET v4( element->value() );
575  ++element;
576 
577  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
578 
579  const size_t jbegin( ( IsUpper<MT1>::value )
580  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
581  :( 0UL ) );
582  const size_t jend( ( IsLower<MT1>::value )
583  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
584  :( N ) );
585  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
586 
587  for( size_t j=jbegin; j<jend; ++j ) {
588  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
589  }
590  }
591  for( ; element!=end; ++element )
592  {
593  const size_t i1( element->index() );
594  const VET v1( element->value() );
595 
596  const size_t jbegin( ( IsUpper<MT1>::value )
597  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
598  :( 0UL ) );
599  const size_t jend( ( IsLower<MT1>::value )
600  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
601  :( N ) );
602  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
603 
604  for( size_t j=jbegin; j<jend; ++j ) {
605  y[j] += v1 * A(i1,j);
606  }
607  }
608  }
610  //**********************************************************************************************
611 
612  //**Vectorized assignment to dense vectors******************************************************
626  template< typename VT1 // Type of the left-hand side target vector
627  , typename VT2 // Type of the left-hand side vector operand
628  , typename MT1 > // Type of the right-hand side matrix operand
630  selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
631  {
633 
634  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
635 
636  constexpr bool remainder( !IsPadded<VT1>::value || !IsPadded<MT1>::value );
637 
638  const size_t N( A.columns() );
639 
640  ConstIterator element( x.begin() );
641  const ConstIterator end( x.end() );
642 
643  const size_t ipos( x.nonZeros() & size_t(-4) );
644  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
645 
646  if( ipos > 3UL )
647  {
648  const size_t i1( element->index() );
649  const VET v1( element->value() );
650  ++element;
651  const size_t i2( element->index() );
652  const VET v2( element->value() );
653  ++element;
654  const size_t i3( element->index() );
655  const VET v3( element->value() );
656  ++element;
657  const size_t i4( element->index() );
658  const VET v4( element->value() );
659  ++element;
660 
661  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
662 
663  const SIMDType xmm1( set( v1 ) );
664  const SIMDType xmm2( set( v2 ) );
665  const SIMDType xmm3( set( v3 ) );
666  const SIMDType xmm4( set( v4 ) );
667 
668  const size_t jpos( remainder ? ( N & size_t(-SIMDSIZE) ) : N );
669  BLAZE_INTERNAL_ASSERT( !remainder || ( N - ( N % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
670 
671  size_t j( 0UL );
672 
673  for( ; j<jpos; j+=SIMDSIZE ) {
674  y.store( j, xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
675  }
676  for( ; remainder && j<N; ++j ) {
677  y[j] = v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
678  }
679  }
680  else
681  {
682  const size_t i1( element->index() );
683  const VET v1( element->value() );
684  ++element;
685 
686  const SIMDType xmm1( set( v1 ) );
687 
688  const size_t jpos( remainder ? ( N & size_t(-SIMDSIZE) ) : N );
689  BLAZE_INTERNAL_ASSERT( !remainder || ( N - ( N % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
690 
691  size_t j( 0UL );
692 
693  for( ; j<jpos; j+=SIMDSIZE ) {
694  y.store( j, xmm1 * A.load(i1,j) );
695  }
696  for( ; remainder && j<N; ++j ) {
697  y[j] = v1 * A(i1,j);
698  }
699  }
700 
701  for( size_t i=(ipos>3UL)?(4UL):(1UL); (i+4UL)<=ipos; i+=4UL )
702  {
703  const size_t i1( element->index() );
704  const VET v1( element->value() );
705  ++element;
706  const size_t i2( element->index() );
707  const VET v2( element->value() );
708  ++element;
709  const size_t i3( element->index() );
710  const VET v3( element->value() );
711  ++element;
712  const size_t i4( element->index() );
713  const VET v4( element->value() );
714  ++element;
715 
716  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
717 
718  const SIMDType xmm1( set( v1 ) );
719  const SIMDType xmm2( set( v2 ) );
720  const SIMDType xmm3( set( v3 ) );
721  const SIMDType xmm4( set( v4 ) );
722 
723  const size_t jbegin( ( IsUpper<MT1>::value )
724  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
725  :( 0UL ) );
726  const size_t jend( ( IsLower<MT1>::value )
727  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
728  :( N ) );
729  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
730 
731  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
732  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
733 
734  size_t j( jbegin );
735 
736  for( ; j<jpos; j+=SIMDSIZE ) {
737  y.store( j, y.load(j) + xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
738  }
739  for( ; remainder && j<jend; ++j ) {
740  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
741  }
742  }
743  for( ; element!=end; ++element )
744  {
745  const size_t i1( element->index() );
746  const VET v1( element->value() );
747 
748  const SIMDType xmm1( set( v1 ) );
749 
750  const size_t jbegin( ( IsUpper<MT1>::value )
751  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
752  :( 0UL ) );
753  const size_t jend( ( IsLower<MT1>::value )
754  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
755  :( N ) );
756  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
757 
758  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
759  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
760 
761  size_t j( jbegin );
762 
763  for( ; j<jpos; j+=SIMDSIZE ) {
764  y.store( j, y.load(j) + xmm1 * A.load(i1,j) );
765  }
766  for( ; remainder && j<jend; ++j ) {
767  y[j] += v1 * A(i1,j);
768  }
769  }
770  }
772  //**********************************************************************************************
773 
774  //**Assignment to sparse vectors****************************************************************
787  template< typename VT2 > // Type of the target sparse vector
788  friend inline void assign( SparseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
789  {
791 
795 
796  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
797 
798  const ResultType tmp( serial( rhs ) );
799  assign( ~lhs, tmp );
800  }
802  //**********************************************************************************************
803 
804  //**Addition assignment to dense vectors********************************************************
816  template< typename VT2 > // Type of the target dense vector
817  friend inline void addAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
818  {
820 
821  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
822 
823  // Evaluation of the left-hand side sparse vector operand
824  LT x( serial( rhs.vec_ ) );
825  if( x.nonZeros() == 0UL ) return;
826 
827  // Evaluation of the right-hand side dense matrix operand
828  RT A( serial( rhs.mat_ ) );
829 
830  // Checking the evaluated operands
831  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
832  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
833  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
834  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
835 
836  // Performing the sparse vector-dense matrix multiplication
837  TSVecDMatMultExpr::selectAddAssignKernel( ~lhs, x, A );
838  }
839  //**********************************************************************************************
840 
841  //**Default addition assignment to dense vectors************************************************
855  template< typename VT1 // Type of the left-hand side target vector
856  , typename VT2 // Type of the left-hand side vector operand
857  , typename MT1 > // Type of the right-hand side matrix operand
859  selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
860  {
862 
863  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
864 
865  const size_t N( A.columns() );
866 
867  ConstIterator element( x.begin() );
868  const ConstIterator end( x.end() );
869 
870  for( ; element!=end; ++element )
871  {
872  const size_t index( element->index() );
873 
875  {
876  y[index] += A(index,index) * element->value();
877  }
878  else
879  {
880  const size_t jbegin( ( IsUpper<MT1>::value )
881  ?( IsStrictlyUpper<MT1>::value ? index+1UL : index )
882  :( 0UL ) );
883  const size_t jend( ( IsLower<MT1>::value )
884  ?( IsStrictlyLower<MT1>::value ? index : index+1UL )
885  :( N ) );
886  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
887 
888  for( size_t j=jbegin; j<jend; ++j ) {
889  y[j] += element->value() * A(index,j);
890  }
891  }
892  }
893  }
895  //**********************************************************************************************
896 
897  //**Optimized addition assignment to dense vectors**********************************************
911  template< typename VT1 // Type of the left-hand side target vector
912  , typename VT2 // Type of the left-hand side vector operand
913  , typename MT1 > // Type of the right-hand side matrix operand
915  selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
916  {
918 
919  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
920 
921  const size_t N( A.columns() );
922 
923  ConstIterator element( x.begin() );
924  const ConstIterator end( x.end() );
925 
926  const size_t ipos( x.nonZeros() & size_t(-4) );
927  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
928 
929  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
930  {
931  const size_t i1( element->index() );
932  const VET v1( element->value() );
933  ++element;
934  const size_t i2( element->index() );
935  const VET v2( element->value() );
936  ++element;
937  const size_t i3( element->index() );
938  const VET v3( element->value() );
939  ++element;
940  const size_t i4( element->index() );
941  const VET v4( element->value() );
942  ++element;
943 
944  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
945 
946  const size_t jbegin( ( IsUpper<MT1>::value )
947  ?( IsStrictlyUpper<MT1>::value ? i+1UL : i1 )
948  :( 0UL ) );
949  const size_t jend( ( IsLower<MT1>::value )
950  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
951  :( N ) );
952  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
953 
954  for( size_t j=jbegin; j<jend; ++j ) {
955  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
956  }
957  }
958  for( ; element!=end; ++element )
959  {
960  const size_t i1( element->index() );
961  const VET v1( element->value() );
962 
963  const size_t jbegin( ( IsUpper<MT1>::value )
964  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
965  :( 0UL ) );
966  const size_t jend( ( IsLower<MT1>::value )
967  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
968  :( N ) );
969  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
970 
971  for( size_t j=jbegin; j<jend; ++j ) {
972  y[j] += v1 * A(i1,j);
973  }
974  }
975  }
977  //**********************************************************************************************
978 
979  //**Vectorized addition assignment to dense vectors*********************************************
993  template< typename VT1 // Type of the left-hand side target vector
994  , typename VT2 // Type of the left-hand side vector operand
995  , typename MT1 > // Type of the right-hand side matrix operand
997  selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
998  {
1000 
1001  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1002 
1003  constexpr bool remainder( !IsPadded<VT1>::value || !IsPadded<MT1>::value );
1004 
1005  const size_t N( A.columns() );
1006 
1007  ConstIterator element( x.begin() );
1008  const ConstIterator end( x.end() );
1009 
1010  const size_t ipos( x.nonZeros() & size_t(-4) );
1011  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1012 
1013  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1014  {
1015  const size_t i1( element->index() );
1016  const VET v1( element->value() );
1017  ++element;
1018  const size_t i2( element->index() );
1019  const VET v2( element->value() );
1020  ++element;
1021  const size_t i3( element->index() );
1022  const VET v3( element->value() );
1023  ++element;
1024  const size_t i4( element->index() );
1025  const VET v4( element->value() );
1026  ++element;
1027 
1028  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1029 
1030  const SIMDType xmm1( set( v1 ) );
1031  const SIMDType xmm2( set( v2 ) );
1032  const SIMDType xmm3( set( v3 ) );
1033  const SIMDType xmm4( set( v4 ) );
1034 
1035  const size_t jbegin( ( IsUpper<MT1>::value )
1036  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1037  :( 0UL ) );
1038  const size_t jend( ( IsLower<MT1>::value )
1039  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
1040  :( N ) );
1041  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1042 
1043  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1044  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1045 
1046  size_t j( jbegin );
1047 
1048  for( ; j<jpos; j+=SIMDSIZE ) {
1049  y.store( j, y.load(j) + xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
1050  }
1051  for( ; remainder && j<jend; ++j ) {
1052  y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1053  }
1054  }
1055  for( ; element!=end; ++element )
1056  {
1057  const size_t i1( element->index() );
1058  const VET v1( element->value() );
1059 
1060  const SIMDType xmm1( set( v1 ) );
1061 
1062  const size_t jbegin( ( IsUpper<MT1>::value )
1063  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1064  :( 0UL ) );
1065  const size_t jend( ( IsLower<MT1>::value )
1066  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
1067  :( N ) );
1068  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1069 
1070  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1071  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1072 
1073  size_t j( jbegin );
1074 
1075  for( ; j<jpos; j+=SIMDSIZE ) {
1076  y.store( j, y.load(j) + xmm1 * A.load(i1,j) );
1077  }
1078  for( ; remainder && j<jend; ++j ) {
1079  y[j] += v1 * A(i1,j);
1080  }
1081  }
1082  }
1084  //**********************************************************************************************
1085 
1086  //**Addition assignment to sparse vectors*******************************************************
1087  // No special implementation for the addition assignment to sparse vectors.
1088  //**********************************************************************************************
1089 
1090  //**Subtraction assignment to dense vectors*****************************************************
1102  template< typename VT2 > // Type of the target dense vector
1103  friend inline void subAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1104  {
1106 
1107  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1108 
1109  // Evaluation of the left-hand side sparse vector operand
1110  LT x( serial( rhs.vec_ ) );
1111  if( x.nonZeros() == 0UL ) return;
1112 
1113  // Evaluation of the right-hand side dense matrix operand
1114  RT A( serial( rhs.mat_ ) );
1115 
1116  // Checking the evaluated operands
1117  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1118  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1119  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1120  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1121 
1122  // Performing the sparse vector-dense matrix multiplication
1123  TSVecDMatMultExpr::selectSubAssignKernel( ~lhs, x, A );
1124  }
1125  //**********************************************************************************************
1126 
1127  //**Default subtraction assignment to dense vectors*********************************************
1141  template< typename VT1 // Type of the left-hand side target vector
1142  , typename VT2 // Type of the left-hand side vector operand
1143  , typename MT1 > // Type of the right-hand side matrix operand
1145  selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1146  {
1148 
1149  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1150 
1151  const size_t N( A.columns() );
1152 
1153  ConstIterator element( x.begin() );
1154  const ConstIterator end( x.end() );
1155 
1156  for( ; element!=end; ++element )
1157  {
1158  const size_t index( element->index() );
1159 
1161  {
1162  y[index] -= A(index,index) * element->value();
1163  }
1164  else
1165  {
1166  const size_t jbegin( ( IsUpper<MT1>::value )
1167  ?( IsStrictlyUpper<MT1>::value ? index+1UL : index )
1168  :( 0UL ) );
1169  const size_t jend( ( IsLower<MT1>::value )
1170  ?( IsStrictlyLower<MT1>::value ? index : index+1UL )
1171  :( N ) );
1172  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1173 
1174  for( size_t j=jbegin; j<jend; ++j ) {
1175  y[j] -= element->value() * A(index,j);
1176  }
1177  }
1178  }
1179  }
1181  //**********************************************************************************************
1182 
1183  //**Optimized subtraction assignment to dense vectors*******************************************
1197  template< typename VT1 // Type of the left-hand side target vector
1198  , typename VT2 // Type of the left-hand side vector operand
1199  , typename MT1 > // Type of the right-hand side matrix operand
1201  selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1202  {
1204 
1205  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1206 
1207  const size_t N( A.columns() );
1208 
1209  ConstIterator element( x.begin() );
1210  const ConstIterator end( x.end() );
1211 
1212  const size_t ipos( x.nonZeros() & size_t(-4) );
1213  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1214 
1215  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1216  {
1217  const size_t i1( element->index() );
1218  const VET v1( element->value() );
1219  ++element;
1220  const size_t i2( element->index() );
1221  const VET v2( element->value() );
1222  ++element;
1223  const size_t i3( element->index() );
1224  const VET v3( element->value() );
1225  ++element;
1226  const size_t i4( element->index() );
1227  const VET v4( element->value() );
1228  ++element;
1229 
1230  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1231 
1232  const size_t jbegin( ( IsUpper<MT1>::value )
1233  ?( IsStrictlyUpper<MT1>::value ? i+1UL : i1 )
1234  :( 0UL ) );
1235  const size_t jend( ( IsLower<MT1>::value )
1236  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
1237  :( N ) );
1238  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1239 
1240  for( size_t j=jbegin; j<jend; ++j ) {
1241  y[j] -= v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1242  }
1243  }
1244  for( ; element!=end; ++element )
1245  {
1246  const size_t i1( element->index() );
1247  const VET v1( element->value() );
1248 
1249  const size_t jbegin( ( IsUpper<MT1>::value )
1250  ?( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 )
1251  :( 0UL ) );
1252  const size_t jend( ( IsLower<MT1>::value )
1253  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
1254  :( N ) );
1255  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1256 
1257  for( size_t j=jbegin; j<jend; ++j ) {
1258  y[j] -= v1 * A(i1,j);
1259  }
1260  }
1261  }
1263  //**********************************************************************************************
1264 
1265  //**Vectorized subtraction assignment to dense vectors******************************************
1279  template< typename VT1 // Type of the left-hand side target vector
1280  , typename VT2 // Type of the left-hand side vector operand
1281  , typename MT1 > // Type of the right-hand side matrix operand
1283  selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1284  {
1286 
1287  BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1288 
1289  constexpr bool remainder( !IsPadded<VT1>::value || !IsPadded<MT1>::value );
1290 
1291  const size_t N( A.columns() );
1292 
1293  ConstIterator element( x.begin() );
1294  const ConstIterator end( x.end() );
1295 
1296  const size_t ipos( x.nonZeros() & size_t(-4) );
1297  BLAZE_INTERNAL_ASSERT( ( x.nonZeros() - ( x.nonZeros() % 4UL ) ) == ipos, "Invalid end calculation" );
1298 
1299  for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1300  {
1301  const size_t i1( element->index() );
1302  const VET v1( element->value() );
1303  ++element;
1304  const size_t i2( element->index() );
1305  const VET v2( element->value() );
1306  ++element;
1307  const size_t i3( element->index() );
1308  const VET v3( element->value() );
1309  ++element;
1310  const size_t i4( element->index() );
1311  const VET v4( element->value() );
1312  ++element;
1313 
1314  BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1315 
1316  const SIMDType xmm1( set( v1 ) );
1317  const SIMDType xmm2( set( v2 ) );
1318  const SIMDType xmm3( set( v3 ) );
1319  const SIMDType xmm4( set( v4 ) );
1320 
1321  const size_t jbegin( ( IsUpper<MT1>::value )
1322  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1323  :( 0UL ) );
1324  const size_t jend( ( IsLower<MT1>::value )
1325  ?( IsStrictlyLower<MT1>::value ? i4 : i4+1UL )
1326  :( N ) );
1327  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1328 
1329  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1330  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1331 
1332  size_t j( jbegin );
1333 
1334  for( ; j<jpos; j+=SIMDSIZE ) {
1335  y.store( j, y.load(j) - xmm1 * A.load(i1,j) - xmm2 * A.load(i2,j) - xmm3 * A.load(i3,j) - xmm4 * A.load(i4,j) );
1336  }
1337  for( ; remainder && j<jend; ++j ) {
1338  y[j] -= v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1339  }
1340  }
1341  for( ; element!=end; ++element )
1342  {
1343  const size_t i1( element->index() );
1344  const VET v1( element->value() );
1345 
1346  const SIMDType xmm1( set( v1 ) );
1347 
1348  const size_t jbegin( ( IsUpper<MT1>::value )
1349  ?( ( IsStrictlyUpper<MT1>::value ? i1+1UL : i1 ) & size_t(-SIMDSIZE) )
1350  :( 0UL ) );
1351  const size_t jend( ( IsLower<MT1>::value )
1352  ?( IsStrictlyLower<MT1>::value ? i1 : i1+1UL )
1353  :( N ) );
1354  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1355 
1356  const size_t jpos( remainder ? ( jend & size_t(-SIMDSIZE) ) : jend );
1357  BLAZE_INTERNAL_ASSERT( !remainder || ( jend - ( jend % SIMDSIZE ) ) == jpos, "Invalid end calculation" );
1358 
1359  size_t j( jbegin );
1360 
1361  for( ; j<jpos; j+=SIMDSIZE ) {
1362  y.store( j, y.load(j) - xmm1 * A.load(i1,j) );
1363  }
1364  for( ; remainder && j<jend; ++j ) {
1365  y[j] -= v1 * A(i1,j);
1366  }
1367  }
1368  }
1370  //**********************************************************************************************
1371 
1372  //**Subtraction assignment to sparse vectors****************************************************
1373  // No special implementation for the subtraction assignment to sparse vectors.
1374  //**********************************************************************************************
1375 
1376  //**Multiplication assignment to dense vectors**************************************************
1388  template< typename VT2 > // Type of the target dense vector
1389  friend inline void multAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1390  {
1392 
1396 
1397  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1398 
1399  const ResultType tmp( serial( rhs ) );
1400  multAssign( ~lhs, tmp );
1401  }
1402  //**********************************************************************************************
1403 
1404  //**Multiplication assignment to sparse vectors*************************************************
1405  // No special implementation for the multiplication assignment to sparse vectors.
1406  //**********************************************************************************************
1407 
1408  //**Division assignment to dense vectors********************************************************
1420  template< typename VT2 > // Type of the target dense vector
1421  friend inline void divAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1422  {
1424 
1428 
1429  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1430 
1431  const ResultType tmp( serial( rhs ) );
1432  divAssign( ~lhs, tmp );
1433  }
1434  //**********************************************************************************************
1435 
1436  //**Division assignment to sparse vectors*******************************************************
1437  // No special implementation for the division assignment to sparse vectors.
1438  //**********************************************************************************************
1439 
1440  //**SMP assignment to dense vectors*************************************************************
1455  template< typename VT2 > // Type of the target dense vector
1456  friend inline EnableIf_< UseSMPAssign<VT2> >
1458  {
1460 
1461  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1462 
1463  // Evaluation of the left-hand side sparse vector operand
1464  LT x( rhs.vec_ );
1465  if( x.nonZeros() == 0UL ) {
1466  reset( ~lhs );
1467  return;
1468  }
1469 
1470  // Evaluation of the right-hand side dense matrix operand
1471  RT A( rhs.mat_ );
1472 
1473  // Checking the evaluated operands
1474  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1475  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1476  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1477  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1478 
1479  // Performing the sparse vector-dense matrix multiplication
1480  smpAssign( ~lhs, x * A );
1481  }
1483  //**********************************************************************************************
1484 
1485  //**SMP assignment to sparse vectors************************************************************
1500  template< typename VT2 > // Type of the target sparse vector
1501  friend inline EnableIf_< UseSMPAssign<VT2> >
1503  {
1505 
1509 
1510  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1511 
1512  const ResultType tmp( rhs );
1513  smpAssign( ~lhs, tmp );
1514  }
1516  //**********************************************************************************************
1517 
1518  //**SMP addition assignment to dense vectors****************************************************
1532  template< typename VT2 > // Type of the target dense vector
1533  friend inline EnableIf_< UseSMPAssign<VT2> >
1535  {
1537 
1538  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1539 
1540  // Evaluation of the left-hand side sparse vector operand
1541  LT x( rhs.vec_ );
1542  if( x.nonZeros() == 0UL ) return;
1543 
1544  // Evaluation of the right-hand side dense matrix operand
1545  RT A( rhs.mat_ );
1546 
1547  // Checking the evaluated operands
1548  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1549  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1550  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1551  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1552 
1553  // Performing the sparse vector-dense matrix multiplication
1554  smpAddAssign( ~lhs, x * A );
1555  }
1556  //**********************************************************************************************
1557 
1558  //**SMP addition assignment to sparse vectors***************************************************
1559  // No special implementation for the SMP addition assignment to sparse vectors.
1560  //**********************************************************************************************
1561 
1562  //**SMP subtraction assignment to dense vectors*************************************************
1576  template< typename VT2 > // Type of the target dense vector
1577  friend inline EnableIf_< UseSMPAssign<VT2> >
1579  {
1581 
1582  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1583 
1584  // Evaluation of the left-hand side sparse vector operand
1585  LT x( rhs.vec_ );
1586  if( x.nonZeros() == 0UL ) return;
1587 
1588  // Evaluation of the right-hand side dense matrix operand
1589  RT A( rhs.mat_ );
1590 
1591  // Checking the evaluated operands
1592  BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1593  BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1594  BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1595  BLAZE_INTERNAL_ASSERT( A.columns() == (~lhs).size() , "Invalid vector size" );
1596 
1597  // Performing the sparse vector-dense matrix multiplication
1598  smpSubAssign( ~lhs, x * A );
1599  }
1600  //**********************************************************************************************
1601 
1602  //**SMP subtraction assignment to sparse vectors************************************************
1603  // No special implementation for the SMP subtraction assignment to sparse vectors.
1604  //**********************************************************************************************
1605 
1606  //**SMP multiplication assignment to dense vectors**********************************************
1620  template< typename VT2 > // Type of the target dense vector
1621  friend inline EnableIf_< UseSMPAssign<VT2> >
1623  {
1625 
1629 
1630  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1631 
1632  const ResultType tmp( rhs );
1633  smpMultAssign( ~lhs, tmp );
1634  }
1635  //**********************************************************************************************
1636 
1637  //**SMP multiplication assignment to sparse vectors*********************************************
1638  // No special implementation for the SMP multiplication assignment to sparse vectors.
1639  //**********************************************************************************************
1640 
1641  //**SMP division assignment to dense vectors****************************************************
1655  template< typename VT2 > // Type of the target dense vector
1656  friend inline EnableIf_< UseSMPAssign<VT2> >
1658  {
1660 
1664 
1665  BLAZE_INTERNAL_ASSERT( (~lhs).size() == rhs.size(), "Invalid vector sizes" );
1666 
1667  const ResultType tmp( rhs );
1668  smpDivAssign( ~lhs, tmp );
1669  }
1670  //**********************************************************************************************
1671 
1672  //**SMP division assignment to sparse vectors***************************************************
1673  // No special implementation for the SMP division assignment to sparse vectors.
1674  //**********************************************************************************************
1675 
1676  //**Compile time checks*************************************************************************
1684  //**********************************************************************************************
1685 };
1686 //*************************************************************************************************
1687 
1688 
1689 
1690 
1691 //=================================================================================================
1692 //
1693 // GLOBAL BINARY ARITHMETIC OPERATORS
1694 //
1695 //=================================================================================================
1696 
1697 //*************************************************************************************************
1728 template< typename VT // Type of the left-hand side sparse vector
1729  , typename MT > // Type of the right-hand side dense matrix
1730 inline decltype(auto)
1731  operator*( const SparseVector<VT,true>& vec, const DenseMatrix<MT,false>& mat )
1732 {
1734 
1736 
1737  if( (~vec).size() != (~mat).rows() ) {
1738  BLAZE_THROW_INVALID_ARGUMENT( "Vector and matrix sizes do not match" );
1739  }
1740 
1741  using ReturnType = const TSVecDMatMultExpr<VT,MT>;
1742  return ReturnType( ~vec, ~mat );
1743 }
1744 //*************************************************************************************************
1745 
1746 
1747 
1748 
1749 //=================================================================================================
1750 //
1751 // GLOBAL RESTRUCTURING BINARY ARITHMETIC OPERATORS
1752 //
1753 //=================================================================================================
1754 
1755 //*************************************************************************************************
1769 template< typename VT // Type of the left-hand side sparse vector
1770  , typename MT > // Matrix base type of the right-hand side expression
1771 inline decltype(auto)
1772  operator*( const SparseVector<VT,true>& vec, const MatMatMultExpr<MT>& mat )
1773 {
1775 
1776  return ( vec * (~mat).leftOperand() ) * (~mat).rightOperand();
1777 }
1779 //*************************************************************************************************
1780 
1781 
1782 
1783 
1784 //=================================================================================================
1785 //
1786 // SIZE SPECIALIZATIONS
1787 //
1788 //=================================================================================================
1789 
1790 //*************************************************************************************************
1792 template< typename VT, typename MT >
1793 struct Size< TSVecDMatMultExpr<VT,MT>, 0UL >
1794  : public Size<MT,1UL>
1795 {};
1797 //*************************************************************************************************
1798 
1799 
1800 
1801 
1802 //=================================================================================================
1803 //
1804 // ISALIGNED SPECIALIZATIONS
1805 //
1806 //=================================================================================================
1807 
1808 //*************************************************************************************************
1810 template< typename VT, typename MT >
1811 struct IsAligned< TSVecDMatMultExpr<VT,MT> >
1812  : public IsAligned<MT>
1813 {};
1815 //*************************************************************************************************
1816 
1817 } // namespace blaze
1818 
1819 #endif
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:329
friend EnableIf_< UseSMPAssign< VT2 > > smpDivAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP division assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ...
Definition: TSVecDMatMultExpr.h:1657
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
friend EnableIf_< UseSMPAssign< VT2 > > smpAddAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP addition assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ...
Definition: TSVecDMatMultExpr.h:1534
Header file for auxiliary alias declarations.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:131
Header file for the blaze::checked and blaze::unchecked instances.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:71
size_t size() const noexcept
Returns the current size/dimension of the vector.
Definition: TSVecDMatMultExpr.h:301
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TSVecDMatMultExpr.h:345
friend void multAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Multiplication assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ...
Definition: TSVecDMatMultExpr.h:1389
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:201
Header file for basic type definitions.
Expression object for transpose sparse vector-dense matrix multiplications.The TSVecDMatMultExpr clas...
Definition: Forward.h:173
friend EnableIf_< UseSMPAssign< VT2 > > smpMultAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP multiplication assignment of a transpose sparse vector-dense matrix multiplication to a dense vec...
Definition: TSVecDMatMultExpr.h:1622
Header file for the serial shim.
Header file for the IsDiagonal type trait.
CompositeType_< MT > MCT
Composite type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:118
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the IsSame and IsStrictlySame type traits.
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:364
Availability of a SIMD multiplication for the given data types.Depending on the available instruction...
Definition: HasSIMDMult.h:172
LeftOperand vec_
Left-hand side sparse vector of the multiplication expression.
Definition: TSVecDMatMultExpr.h:372
typename SIMDTrait< T >::Type SIMDTrait_
Auxiliary alias declaration for the SIMDTrait class template.The SIMDTrait_ alias declaration provide...
Definition: SIMDTrait.h:316
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:588
ElementType_< VRT > VET
Element type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:115
ElementType_< ResultType > ElementType
Resulting element type.
Definition: TSVecDMatMultExpr.h:199
constexpr Unchecked unchecked
Global Unchecked instance.The blaze::unchecked instance is an optional token for the creation of view...
Definition: Check.h:138
IfTrue_< evaluateVector, const VRT, VCT > LT
Type for the assignment of the left-hand side sparse vector operand.
Definition: TSVecDMatMultExpr.h:211
Header file for the DenseVector base class.
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:87
Availability of a SIMD addition for the given data types.Depending on the available instruction set (...
Definition: HasSIMDAdd.h:171
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TSVecDMatMultExpr.h:333
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:291
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:87
Constraints on the storage order of matrix types.
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:198
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Compile time check for data types.This type trait tests whether or not the given types can be combine...
Definition: IsSIMDCombinable.h:120
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:343
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:80
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
Compile time check for the alignment of data types.This type trait tests whether the given data type ...
Definition: IsAligned.h:87
Constraint on the data type.
RightOperand mat_
Right-hand side dense matrix of the multiplication expression.
Definition: TSVecDMatMultExpr.h:373
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:71
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
MultTrait_< VRT, MRT > ResultType
Result type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:197
Header file for the DisableIf class template.
friend void addAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Addition assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:817
Header file for the multiplication trait.
RightOperand rightOperand() const noexcept
Returns the right-hand side dense matrix operand.
Definition: TSVecDMatMultExpr.h:321
Header file for the IsStrictlyUpper type trait.
LeftOperand leftOperand() const noexcept
Returns the left-hand side sparse vector operand.
Definition: TSVecDMatMultExpr.h:311
Namespace of the Blaze C++ math library.
Definition: Blaze.h:58
Header file for the If class template.
friend void divAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Division assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:1421
Compile time check for data types with padding.This type trait tests whether the given data type empl...
Definition: IsPadded.h:76
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:102
const ResultType CompositeType
Data type for composite expression templates.
Definition: TSVecDMatMultExpr.h:202
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the HasSIMDAdd type trait.
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:3085
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for all SIMD functionality.
Base class for N-dimensional dense vectors.The DenseVector class is a base class for all arbitrarily ...
Definition: DenseVector.h:76
ResultType_< MT > MRT
Result type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:114
Header file for the IsLower type trait.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional vector type...
Definition: SparseVector.h:61
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:89
If_< IsExpression< VT >, const VT, const VT &> LeftOperand
Composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:205
#define BLAZE_CONSTRAINT_MUST_NOT_BE_MATMATMULTEXPR_TYPE(T)
Constraint on the data type.In case the given data type T is a matrix/matrix multiplication expressio...
Definition: MatMatMultExpr.h:87
ElementType_< MRT > MET
Element type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:116
Constraint on the data type.
Header file for the exception macros of the math module.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:430
decltype(auto) operator*(const DenseMatrix< MT1, false > &lhs, const DenseMatrix< MT2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:8893
Constraint on the data type.
Header file for all forward declarations for expression class templates.
Constraint on the data type.
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
Header file for the IsPadded type trait.
IfTrue_< evaluateMatrix, const MRT, MCT > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: TSVecDMatMultExpr.h:214
Compile time check for resizable data types.This type trait tests whether the given data type is a re...
Definition: IsResizable.h:75
ResultType_< VT > VRT
Result type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:113
Base class for all matrix/matrix multiplication expression templates.The MatMatMultExpr class serves ...
Definition: MatMatMultExpr.h:67
Header file for the IsSIMDCombinable type trait.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TSVecDMatMultExpr.h:355
Header file for run time assertion macros.
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:154
Header file for the reset shim.
Constraint on the data type.
SIMD characteristics of data types.The SIMDTrait class template provides the SIMD characteristics of ...
Definition: SIMDTrait.h:296
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:94
ReturnType operator[](size_t index) const
Subscript operator for the direct access to the vector elements.
Definition: TSVecDMatMultExpr.h:254
Header file for the TVecMatMultExpr base class.
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:816
TSVecDMatMultExpr(const VT &vec, const MT &mat) noexcept
Constructor for the TSVecDMatMultExpr class.
Definition: TSVecDMatMultExpr.h:240
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
SIMDTrait_< ElementType > SIMDType
Resulting SIMD element type.
Definition: TSVecDMatMultExpr.h:200
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TSVecDMatMultExpr.h:365
Header file for the RemoveReference type trait.
typename EnableIf< Condition, T >::Type EnableIf_
Auxiliary alias declaration for the EnableIf class template.The EnableIf_ alias declaration provides ...
Definition: EnableIf.h:224
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
BLAZE_ALWAYS_INLINE size_t rows(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of rows of the matrix.
Definition: Matrix.h:490
const Type & ReturnType
Return type for expression template evaluations.
Definition: CompressedMatrix.h:3080
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional vector type...
Definition: DenseVector.h:61
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_TVECMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid vector/matrix ...
Definition: TVecMatMultExpr.h:108
Header file for the IsComputation type trait class.
Compile time evaluation of the size of vectors and matrices.The Size type trait evaluates the size of...
Definition: Size.h:80
Base class for sparse vectors.The SparseVector class is a base class for all arbitrarily sized (N-dim...
Definition: Forward.h:130
#define BLAZE_CONSTRAINT_MUST_BE_ROW_VECTOR_TYPE(T)
Constraint on the data type.In case the given data type T is not a row dense or sparse vector type (i...
Definition: RowVector.h:61
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:423
friend void subAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Subtraction assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:1103
Header file for the IsUpper type trait.
Constraint on the data type.
ReturnType at(size_t index) const
Checked access to the vector elements.
Definition: TSVecDMatMultExpr.h:288
friend EnableIf_< UseSMPAssign< VT2 > > smpSubAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
SMP subtraction assignment of a transpose sparse vector-dense matrix multiplication to a dense vector...
Definition: TSVecDMatMultExpr.h:1578
Header file for the IsResizable type trait.
Header file for the Size type trait.
Size type of the Blaze library.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
If_< IsExpression< MT >, const MT, const MT &> RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TSVecDMatMultExpr.h:208
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
CompositeType_< VT > VCT
Composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:117
Constraint on the transpose flag of vector types.
Header file for the IsExpression type trait class.
Header file for the function trace functionality.