Blaze 3.9
TSVecDMatMultExpr.h
Go to the documentation of this file.
1//=================================================================================================
33//=================================================================================================
34
35#ifndef _BLAZE_MATH_EXPRESSIONS_TSVECDMATMULTEXPR_H_
36#define _BLAZE_MATH_EXPRESSIONS_TSVECDMATMULTEXPR_H_
37
38
39//*************************************************************************************************
40// Includes
41//*************************************************************************************************
42
43#include <blaze/math/Aliases.h>
62#include <blaze/math/SIMD.h>
83#include <blaze/util/Assert.h>
84#include <blaze/util/EnableIf.h>
87#include <blaze/util/mpl/If.h>
88#include <blaze/util/Types.h>
89
90
91namespace blaze {
92
93//=================================================================================================
94//
95// CLASS TSVECDMATMULTEXPR
96//
97//=================================================================================================
98
99//*************************************************************************************************
106template< typename VT // Type of the left-hand side sparse vector
107 , typename MT > // Type of the right-hand side dense matrix
109 : public TVecMatMultExpr< DenseVector< TSVecDMatMultExpr<VT,MT>, true > >
110 , private Computation
111{
112 private:
113 //**Type definitions****************************************************************************
120 //**********************************************************************************************
121
122 //**********************************************************************************************
124 static constexpr bool evaluateVector = ( IsComputation_v<VT> || RequiresEvaluation_v<VT> );
125 //**********************************************************************************************
126
127 //**********************************************************************************************
129 static constexpr bool evaluateMatrix = RequiresEvaluation_v<MT>;
130 //**********************************************************************************************
131
132 //**********************************************************************************************
134
138 template< typename T1 >
139 static constexpr bool UseSMPAssign_v = ( evaluateVector || evaluateMatrix );
141 //**********************************************************************************************
142
143 //**********************************************************************************************
145
149 template< typename T1, typename T2, typename T3 >
150 static constexpr bool UseVectorizedKernel_v =
151 ( useOptimizedKernels &&
152 !IsDiagonal_v<T3> &&
153 T1::simdEnabled && T3::simdEnabled &&
154 IsSIMDCombinable_v< ElementType_t<T1>
156 , ElementType_t<T3> > &&
157 HasSIMDAdd_v< ElementType_t<T2>, ElementType_t<T3> > &&
158 HasSIMDMult_v< ElementType_t<T2>, ElementType_t<T3> > );
160 //**********************************************************************************************
161
162 //**********************************************************************************************
164
168 template< typename T1, typename T2, typename T3 >
169 static constexpr bool UseOptimizedKernel_v =
170 ( useOptimizedKernels &&
171 !UseVectorizedKernel_v<T1,T2,T3> &&
172 !IsDiagonal_v<T3> &&
173 !IsResizable_v< ElementType_t<T1> > &&
174 !IsResizable_v<VET> );
176 //**********************************************************************************************
177
178 //**********************************************************************************************
180
183 template< typename T1, typename T2, typename T3 >
184 static constexpr bool UseDefaultKernel_v =
185 ( !UseVectorizedKernel_v<T1,T2,T3> && !UseOptimizedKernel_v<T1,T2,T3> );
187 //**********************************************************************************************
188
189 public:
190 //**Type definitions****************************************************************************
193
196
201 using ReturnType = const ElementType;
202 using CompositeType = const ResultType;
203
205 using LeftOperand = If_t< IsExpression_v<VT>, const VT, const VT& >;
206
208 using RightOperand = If_t< IsExpression_v<MT>, const MT, const MT& >;
209
212
215 //**********************************************************************************************
216
217 //**Compilation flags***************************************************************************
219 static constexpr bool simdEnabled =
220 ( !IsDiagonal_v<MT> &&
221 MT::simdEnabled &&
222 HasSIMDAdd_v<VET,MET> &&
223 HasSIMDMult_v<VET,MET> );
224
226 static constexpr bool smpAssignable =
227 ( !evaluateVector && VT::smpAssignable && !evaluateMatrix && MT::smpAssignable );
228 //**********************************************************************************************
229
230 //**SIMD properties*****************************************************************************
232 static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
233 //**********************************************************************************************
234
235 //**Constructor*********************************************************************************
241 inline TSVecDMatMultExpr( const VT& vec, const MT& mat ) noexcept
242 : vec_( vec ) // Left-hand side sparse vector of the multiplication expression
243 , mat_( mat ) // Right-hand side dense matrix of the multiplication expression
244 {
245 BLAZE_INTERNAL_ASSERT( vec_.size() == mat_.rows(), "Invalid vector and matrix sizes" );
246 }
247 //**********************************************************************************************
248
249 //**Subscript operator**************************************************************************
255 inline ReturnType operator[]( size_t index ) const {
256 BLAZE_INTERNAL_ASSERT( index < mat_.columns(), "Invalid vector access index" );
257
258 if( IsDiagonal_v<MT> )
259 {
260 return vec_[index] * mat_(index,index);
261 }
262 else if( IsLower_v<MT> )
263 {
264 const size_t begin( IsStrictlyLower_v<MT> ? index+1UL : index );
265 const size_t n ( mat_.rows() - begin );
266 return subvector( vec_, begin, n, unchecked ) *
267 subvector( column( mat_, index, unchecked ), begin, n, unchecked );
268 }
269 else if( IsUpper_v<MT> )
270 {
271 const size_t n( IsStrictlyUpper_v<MT> ? index : index+1UL );
272 return subvector( vec_, 0UL, n, unchecked ) *
273 subvector( column( mat_, index, unchecked ), 0UL, n, unchecked );
274 }
275 else
276 {
277 return vec_ * column( mat_, index, unchecked );
278 }
279 }
280 //**********************************************************************************************
281
282 //**At function*********************************************************************************
289 inline ReturnType at( size_t index ) const {
290 if( index >= mat_.columns() ) {
291 BLAZE_THROW_OUT_OF_RANGE( "Invalid vector access index" );
292 }
293 return (*this)[index];
294 }
295 //**********************************************************************************************
296
297 //**Size function*******************************************************************************
302 inline size_t size() const noexcept {
303 return mat_.columns();
304 }
305 //**********************************************************************************************
306
307 //**Left operand access*************************************************************************
312 inline LeftOperand leftOperand() const noexcept {
313 return vec_;
314 }
315 //**********************************************************************************************
316
317 //**Right operand access************************************************************************
322 inline RightOperand rightOperand() const noexcept {
323 return mat_;
324 }
325 //**********************************************************************************************
326
327 //**********************************************************************************************
333 template< typename T >
334 inline bool canAlias( const T* alias ) const noexcept {
335 return vec_.isAliased( alias ) || mat_.isAliased( alias );
336 }
337 //**********************************************************************************************
338
339 //**********************************************************************************************
345 template< typename T >
346 inline bool isAliased( const T* alias ) const noexcept {
347 return vec_.isAliased( alias ) || mat_.isAliased( alias );
348 }
349 //**********************************************************************************************
350
351 //**********************************************************************************************
356 inline bool isAligned() const noexcept {
357 return mat_.isAligned();
358 }
359 //**********************************************************************************************
360
361 //**********************************************************************************************
366 inline bool canSMPAssign() const noexcept {
367 return ( size() > SMP_TSVECDMATMULT_THRESHOLD );
368 }
369 //**********************************************************************************************
370
371 private:
372 //**Member variables****************************************************************************
375 //**********************************************************************************************
376
377 //**Assignment to dense vectors*****************************************************************
390 template< typename VT2 > // Type of the target dense vector
391 friend inline void assign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
392 {
394
395 BLAZE_INTERNAL_ASSERT( (*lhs).size() == rhs.size(), "Invalid vector sizes" );
396
397 // Evaluation of the left-hand side sparse vector operand
398 LT x( serial( rhs.vec_ ) );
399 if( x.nonZeros() == 0UL ) {
400 reset( *lhs );
401 return;
402 }
403
404 // Evaluation of the right-hand side dense matrix operand
405 RT A( serial( rhs.mat_ ) );
406
407 // Checking the evaluated operands
408 BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
409 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
410 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
411 BLAZE_INTERNAL_ASSERT( A.columns() == (*lhs).size() , "Invalid vector size" );
412
413 // Performing the sparse vector-dense matrix multiplication
414 TSVecDMatMultExpr::selectAssignKernel( *lhs, x, A );
415 }
417 //**********************************************************************************************
418
419 //**Default assignment to dense vectors*********************************************************
433 template< typename VT1 // Type of the left-hand side target vector
434 , typename VT2 // Type of the left-hand side vector operand
435 , typename MT1 > // Type of the right-hand side matrix operand
436 static inline auto selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
438 {
439 BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
440
441 const size_t N( A.columns() );
442
443 auto element( x.begin() );
444 const auto end( x.end() );
445
446 size_t last( 0UL );
447
448 if( IsUpper_v<MT1> ) {
449 const size_t jend( IsStrictlyUpper_v<MT1> ? element->index()+1UL : element->index() );
450 for( size_t j=0UL; j<jend; ++j )
451 reset( y[j] );
452 }
453
454 for( ; element!=end; ++element )
455 {
456 const size_t index( element->index() );
457
458 if( IsDiagonal_v<MT1> )
459 {
460 for( size_t j=last; j<index; ++j )
461 reset( y[j] );
462
463 y[index] = element->value() * A(index,index);
464 last = index + 1UL;
465 }
466 else
467 {
468 const size_t jbegin( ( IsUpper_v<MT1> )
469 ?( IsStrictlyUpper_v<MT1> ? index+1UL : index )
470 :( 0UL ) );
471 const size_t jend( ( IsLower_v<MT1> )
472 ?( IsStrictlyLower_v<MT1> ? index : index+1UL )
473 :( N ) );
474 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
475
476 for( size_t j=jbegin; j<last; ++j ) {
477 y[j] += element->value() * A(index,j);
478 }
479 for( size_t j=last; j<jend; ++j ) {
480 y[j] = element->value() * A(index,j);
481 }
482
483 last = jend;
484 }
485 }
486
487 if( IsLower_v<MT1> ) {
488 for( size_t j=last; j<N; ++j )
489 reset( y[j] );
490 }
491 }
493 //**********************************************************************************************
494
495 //**Optimized assignment to dense vectors*******************************************************
509 template< typename VT1 // Type of the left-hand side target vector
510 , typename VT2 // Type of the left-hand side vector operand
511 , typename MT1 > // Type of the right-hand side matrix operand
512 static inline auto selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
513 -> EnableIf_t< UseOptimizedKernel_v<VT1,VT2,MT1> >
514 {
515 BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
516
517 const size_t N( A.columns() );
518
519 auto element( x.begin() );
520 const auto end( x.end() );
521
522 const size_t ipos( prevMultiple( x.nonZeros(), 4UL ) );
523 BLAZE_INTERNAL_ASSERT( ipos <= x.nonZeros(), "Invalid end calculation" );
524
525 if( ipos > 3UL )
526 {
527 const size_t i1( element->index() );
528 const VET v1( element->value() );
529 ++element;
530 const size_t i2( element->index() );
531 const VET v2( element->value() );
532 ++element;
533 const size_t i3( element->index() );
534 const VET v3( element->value() );
535 ++element;
536 const size_t i4( element->index() );
537 const VET v4( element->value() );
538 ++element;
539
540 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
541
542 for( size_t j=0UL; j<N; ++j ) {
543 y[j] = v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
544 }
545 }
546 else
547 {
548 const size_t i1( element->index() );
549 const VET v1( element->value() );
550 ++element;
551
552 for( size_t j=0UL; j<N; ++j ) {
553 y[j] = v1 * A(i1,j);
554 }
555 }
556
557 for( size_t i=(ipos>3UL)?(4UL):(1UL); (i+4UL)<=ipos; i+=4UL )
558 {
559 const size_t i1( element->index() );
560 const VET v1( element->value() );
561 ++element;
562 const size_t i2( element->index() );
563 const VET v2( element->value() );
564 ++element;
565 const size_t i3( element->index() );
566 const VET v3( element->value() );
567 ++element;
568 const size_t i4( element->index() );
569 const VET v4( element->value() );
570 ++element;
571
572 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
573
574 const size_t jbegin( ( IsUpper_v<MT1> )
575 ?( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 )
576 :( 0UL ) );
577 const size_t jend( ( IsLower_v<MT1> )
578 ?( IsStrictlyLower_v<MT1> ? i4 : i4+1UL )
579 :( N ) );
580 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
581
582 for( size_t j=jbegin; j<jend; ++j ) {
583 y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
584 }
585 }
586 for( ; element!=end; ++element )
587 {
588 const size_t i1( element->index() );
589 const VET v1( element->value() );
590
591 const size_t jbegin( ( IsUpper_v<MT1> )
592 ?( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 )
593 :( 0UL ) );
594 const size_t jend( ( IsLower_v<MT1> )
595 ?( IsStrictlyLower_v<MT1> ? i1 : i1+1UL )
596 :( N ) );
597 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
598
599 for( size_t j=jbegin; j<jend; ++j ) {
600 y[j] += v1 * A(i1,j);
601 }
602 }
603 }
605 //**********************************************************************************************
606
607 //**Vectorized assignment to dense vectors******************************************************
621 template< typename VT1 // Type of the left-hand side target vector
622 , typename VT2 // Type of the left-hand side vector operand
623 , typename MT1 > // Type of the right-hand side matrix operand
624 static inline auto selectAssignKernel( VT1& y, const VT2& x, const MT1& A )
625 -> EnableIf_t< UseVectorizedKernel_v<VT1,VT2,MT1> >
626 {
627 BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
628
629 constexpr bool remainder( !IsPadded_v<VT1> || !IsPadded_v<MT1> );
630
631 const size_t N( A.columns() );
632
633 auto element( x.begin() );
634 const auto end( x.end() );
635
636 const size_t ipos( prevMultiple( x.nonZeros(), 4UL ) );
637 BLAZE_INTERNAL_ASSERT( ipos <= x.nonZeros(), "Invalid end calculation" );
638
639 if( ipos > 3UL )
640 {
641 const size_t i1( element->index() );
642 const VET v1( element->value() );
643 ++element;
644 const size_t i2( element->index() );
645 const VET v2( element->value() );
646 ++element;
647 const size_t i3( element->index() );
648 const VET v3( element->value() );
649 ++element;
650 const size_t i4( element->index() );
651 const VET v4( element->value() );
652 ++element;
653
654 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
655
656 const SIMDType xmm1( set( v1 ) );
657 const SIMDType xmm2( set( v2 ) );
658 const SIMDType xmm3( set( v3 ) );
659 const SIMDType xmm4( set( v4 ) );
660
661 const size_t jpos( remainder ? prevMultiple( N, SIMDSIZE ) : N );
662 BLAZE_INTERNAL_ASSERT( jpos <= N, "Invalid end calculation" );
663
664 size_t j( 0UL );
665
666 for( ; j<jpos; j+=SIMDSIZE ) {
667 y.store( j, xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
668 }
669 for( ; remainder && j<N; ++j ) {
670 y[j] = v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
671 }
672 }
673 else
674 {
675 const size_t i1( element->index() );
676 const VET v1( element->value() );
677 ++element;
678
679 const SIMDType xmm1( set( v1 ) );
680
681 const size_t jpos( remainder ? prevMultiple( N, SIMDSIZE ) : N );
682 BLAZE_INTERNAL_ASSERT( jpos <= N, "Invalid end calculation" );
683
684 size_t j( 0UL );
685
686 for( ; j<jpos; j+=SIMDSIZE ) {
687 y.store( j, xmm1 * A.load(i1,j) );
688 }
689 for( ; remainder && j<N; ++j ) {
690 y[j] = v1 * A(i1,j);
691 }
692 }
693
694 for( size_t i=(ipos>3UL)?(4UL):(1UL); (i+4UL)<=ipos; i+=4UL )
695 {
696 const size_t i1( element->index() );
697 const VET v1( element->value() );
698 ++element;
699 const size_t i2( element->index() );
700 const VET v2( element->value() );
701 ++element;
702 const size_t i3( element->index() );
703 const VET v3( element->value() );
704 ++element;
705 const size_t i4( element->index() );
706 const VET v4( element->value() );
707 ++element;
708
709 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
710
711 const SIMDType xmm1( set( v1 ) );
712 const SIMDType xmm2( set( v2 ) );
713 const SIMDType xmm3( set( v3 ) );
714 const SIMDType xmm4( set( v4 ) );
715
716 const size_t jbegin( ( IsUpper_v<MT1> )
717 ?( prevMultiple( ( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 ), SIMDSIZE ) )
718 :( 0UL ) );
719 const size_t jend( ( IsLower_v<MT1> )
720 ?( IsStrictlyLower_v<MT1> ? i4 : i4+1UL )
721 :( N ) );
722 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
723
724 const size_t jpos( remainder ? prevMultiple( jend, SIMDSIZE ) : jend );
725 BLAZE_INTERNAL_ASSERT( jpos <= jend, "Invalid end calculation" );
726
727 size_t j( jbegin );
728
729 for( ; j<jpos; j+=SIMDSIZE ) {
730 y.store( j, y.load(j) + xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
731 }
732 for( ; remainder && j<jend; ++j ) {
733 y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
734 }
735 }
736 for( ; element!=end; ++element )
737 {
738 const size_t i1( element->index() );
739 const VET v1( element->value() );
740
741 const SIMDType xmm1( set( v1 ) );
742
743 const size_t jbegin( ( IsUpper_v<MT1> )
744 ?( prevMultiple( ( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 ), SIMDSIZE ) )
745 :( 0UL ) );
746 const size_t jend( ( IsLower_v<MT1> )
747 ?( IsStrictlyLower_v<MT1> ? i1 : i1+1UL )
748 :( N ) );
749 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
750
751 const size_t jpos( remainder ? prevMultiple( jend, SIMDSIZE ) : jend );
752 BLAZE_INTERNAL_ASSERT( jpos <= jend, "Invalid end calculation" );
753
754 size_t j( jbegin );
755
756 for( ; j<jpos; j+=SIMDSIZE ) {
757 y.store( j, y.load(j) + xmm1 * A.load(i1,j) );
758 }
759 for( ; remainder && j<jend; ++j ) {
760 y[j] += v1 * A(i1,j);
761 }
762 }
763 }
765 //**********************************************************************************************
766
767 //**Assignment to sparse vectors****************************************************************
780 template< typename VT2 > // Type of the target sparse vector
781 friend inline void assign( SparseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
782 {
784
788
789 BLAZE_INTERNAL_ASSERT( (*lhs).size() == rhs.size(), "Invalid vector sizes" );
790
791 const ResultType tmp( serial( rhs ) );
792 assign( *lhs, tmp );
793 }
795 //**********************************************************************************************
796
797 //**Addition assignment to dense vectors********************************************************
809 template< typename VT2 > // Type of the target dense vector
810 friend inline void addAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
811 {
813
814 BLAZE_INTERNAL_ASSERT( (*lhs).size() == rhs.size(), "Invalid vector sizes" );
815
816 // Evaluation of the left-hand side sparse vector operand
817 LT x( serial( rhs.vec_ ) );
818 if( x.nonZeros() == 0UL ) return;
819
820 // Evaluation of the right-hand side dense matrix operand
821 RT A( serial( rhs.mat_ ) );
822
823 // Checking the evaluated operands
824 BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
825 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
826 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
827 BLAZE_INTERNAL_ASSERT( A.columns() == (*lhs).size() , "Invalid vector size" );
828
829 // Performing the sparse vector-dense matrix multiplication
830 TSVecDMatMultExpr::selectAddAssignKernel( *lhs, x, A );
831 }
832 //**********************************************************************************************
833
834 //**Default addition assignment to dense vectors************************************************
848 template< typename VT1 // Type of the left-hand side target vector
849 , typename VT2 // Type of the left-hand side vector operand
850 , typename MT1 > // Type of the right-hand side matrix operand
851 static inline auto selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
853 {
854 BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
855
856 const size_t N( A.columns() );
857
858 auto element( x.begin() );
859 const auto end( x.end() );
860
861 for( ; element!=end; ++element )
862 {
863 const size_t index( element->index() );
864
865 if( IsDiagonal_v<MT1> )
866 {
867 y[index] += A(index,index) * element->value();
868 }
869 else
870 {
871 const size_t jbegin( ( IsUpper_v<MT1> )
872 ?( IsStrictlyUpper_v<MT1> ? index+1UL : index )
873 :( 0UL ) );
874 const size_t jend( ( IsLower_v<MT1> )
875 ?( IsStrictlyLower_v<MT1> ? index : index+1UL )
876 :( N ) );
877 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
878
879 for( size_t j=jbegin; j<jend; ++j ) {
880 y[j] += element->value() * A(index,j);
881 }
882 }
883 }
884 }
886 //**********************************************************************************************
887
888 //**Optimized addition assignment to dense vectors**********************************************
902 template< typename VT1 // Type of the left-hand side target vector
903 , typename VT2 // Type of the left-hand side vector operand
904 , typename MT1 > // Type of the right-hand side matrix operand
905 static inline auto selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
906 -> EnableIf_t< UseOptimizedKernel_v<VT1,VT2,MT1> >
907 {
908 BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
909
910 const size_t N( A.columns() );
911
912 auto element( x.begin() );
913 const auto end( x.end() );
914
915 const size_t ipos( prevMultiple( x.nonZeros(), 4UL ) );
916 BLAZE_INTERNAL_ASSERT( ipos <= x.nonZeros(), "Invalid end calculation" );
917
918 for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
919 {
920 const size_t i1( element->index() );
921 const VET v1( element->value() );
922 ++element;
923 const size_t i2( element->index() );
924 const VET v2( element->value() );
925 ++element;
926 const size_t i3( element->index() );
927 const VET v3( element->value() );
928 ++element;
929 const size_t i4( element->index() );
930 const VET v4( element->value() );
931 ++element;
932
933 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
934
935 const size_t jbegin( ( IsUpper_v<MT1> )
936 ?( IsStrictlyUpper_v<MT1> ? i+1UL : i1 )
937 :( 0UL ) );
938 const size_t jend( ( IsLower_v<MT1> )
939 ?( IsStrictlyLower_v<MT1> ? i4 : i4+1UL )
940 :( N ) );
941 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
942
943 for( size_t j=jbegin; j<jend; ++j ) {
944 y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
945 }
946 }
947 for( ; element!=end; ++element )
948 {
949 const size_t i1( element->index() );
950 const VET v1( element->value() );
951
952 const size_t jbegin( ( IsUpper_v<MT1> )
953 ?( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 )
954 :( 0UL ) );
955 const size_t jend( ( IsLower_v<MT1> )
956 ?( IsStrictlyLower_v<MT1> ? i1 : i1+1UL )
957 :( N ) );
958 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
959
960 for( size_t j=jbegin; j<jend; ++j ) {
961 y[j] += v1 * A(i1,j);
962 }
963 }
964 }
966 //**********************************************************************************************
967
968 //**Vectorized addition assignment to dense vectors*********************************************
982 template< typename VT1 // Type of the left-hand side target vector
983 , typename VT2 // Type of the left-hand side vector operand
984 , typename MT1 > // Type of the right-hand side matrix operand
985 static inline auto selectAddAssignKernel( VT1& y, const VT2& x, const MT1& A )
986 -> EnableIf_t< UseVectorizedKernel_v<VT1,VT2,MT1> >
987 {
988 BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
989
990 constexpr bool remainder( !IsPadded_v<VT1> || !IsPadded_v<MT1> );
991
992 const size_t N( A.columns() );
993
994 auto element( x.begin() );
995 const auto end( x.end() );
996
997 const size_t ipos( prevMultiple( x.nonZeros(), 4UL ) );
998 BLAZE_INTERNAL_ASSERT( ipos <= x.nonZeros(), "Invalid end calculation" );
999
1000 for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1001 {
1002 const size_t i1( element->index() );
1003 const VET v1( element->value() );
1004 ++element;
1005 const size_t i2( element->index() );
1006 const VET v2( element->value() );
1007 ++element;
1008 const size_t i3( element->index() );
1009 const VET v3( element->value() );
1010 ++element;
1011 const size_t i4( element->index() );
1012 const VET v4( element->value() );
1013 ++element;
1014
1015 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1016
1017 const SIMDType xmm1( set( v1 ) );
1018 const SIMDType xmm2( set( v2 ) );
1019 const SIMDType xmm3( set( v3 ) );
1020 const SIMDType xmm4( set( v4 ) );
1021
1022 const size_t jbegin( ( IsUpper_v<MT1> )
1023 ?( prevMultiple( ( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 ), SIMDSIZE ) )
1024 :( 0UL ) );
1025 const size_t jend( ( IsLower_v<MT1> )
1026 ?( IsStrictlyLower_v<MT1> ? i4 : i4+1UL )
1027 :( N ) );
1028 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1029
1030 const size_t jpos( remainder ? prevMultiple( jend, SIMDSIZE ) : jend );
1031 BLAZE_INTERNAL_ASSERT( jpos <= jend, "Invalid end calculation" );
1032
1033 size_t j( jbegin );
1034
1035 for( ; j<jpos; j+=SIMDSIZE ) {
1036 y.store( j, y.load(j) + xmm1 * A.load(i1,j) + xmm2 * A.load(i2,j) + xmm3 * A.load(i3,j) + xmm4 * A.load(i4,j) );
1037 }
1038 for( ; remainder && j<jend; ++j ) {
1039 y[j] += v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1040 }
1041 }
1042 for( ; element!=end; ++element )
1043 {
1044 const size_t i1( element->index() );
1045 const VET v1( element->value() );
1046
1047 const SIMDType xmm1( set( v1 ) );
1048
1049 const size_t jbegin( ( IsUpper_v<MT1> )
1050 ?( prevMultiple( ( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 ), SIMDSIZE ) )
1051 :( 0UL ) );
1052 const size_t jend( ( IsLower_v<MT1> )
1053 ?( IsStrictlyLower_v<MT1> ? i1 : i1+1UL )
1054 :( N ) );
1055 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1056
1057 const size_t jpos( remainder ? prevMultiple( jend, SIMDSIZE ) : jend );
1058 BLAZE_INTERNAL_ASSERT( jpos <= jend, "Invalid end calculation" );
1059
1060 size_t j( jbegin );
1061
1062 for( ; j<jpos; j+=SIMDSIZE ) {
1063 y.store( j, y.load(j) + xmm1 * A.load(i1,j) );
1064 }
1065 for( ; remainder && j<jend; ++j ) {
1066 y[j] += v1 * A(i1,j);
1067 }
1068 }
1069 }
1071 //**********************************************************************************************
1072
1073 //**Addition assignment to sparse vectors*******************************************************
1074 // No special implementation for the addition assignment to sparse vectors.
1075 //**********************************************************************************************
1076
1077 //**Subtraction assignment to dense vectors*****************************************************
1089 template< typename VT2 > // Type of the target dense vector
1090 friend inline void subAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1091 {
1093
1094 BLAZE_INTERNAL_ASSERT( (*lhs).size() == rhs.size(), "Invalid vector sizes" );
1095
1096 // Evaluation of the left-hand side sparse vector operand
1097 LT x( serial( rhs.vec_ ) );
1098 if( x.nonZeros() == 0UL ) return;
1099
1100 // Evaluation of the right-hand side dense matrix operand
1101 RT A( serial( rhs.mat_ ) );
1102
1103 // Checking the evaluated operands
1104 BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1105 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1106 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1107 BLAZE_INTERNAL_ASSERT( A.columns() == (*lhs).size() , "Invalid vector size" );
1108
1109 // Performing the sparse vector-dense matrix multiplication
1110 TSVecDMatMultExpr::selectSubAssignKernel( *lhs, x, A );
1111 }
1112 //**********************************************************************************************
1113
1114 //**Default subtraction assignment to dense vectors*********************************************
1128 template< typename VT1 // Type of the left-hand side target vector
1129 , typename VT2 // Type of the left-hand side vector operand
1130 , typename MT1 > // Type of the right-hand side matrix operand
1131 static inline auto selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1133 {
1134 BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1135
1136 const size_t N( A.columns() );
1137
1138 auto element( x.begin() );
1139 const auto end( x.end() );
1140
1141 for( ; element!=end; ++element )
1142 {
1143 const size_t index( element->index() );
1144
1145 if( IsDiagonal_v<MT1> )
1146 {
1147 y[index] -= A(index,index) * element->value();
1148 }
1149 else
1150 {
1151 const size_t jbegin( ( IsUpper_v<MT1> )
1152 ?( IsStrictlyUpper_v<MT1> ? index+1UL : index )
1153 :( 0UL ) );
1154 const size_t jend( ( IsLower_v<MT1> )
1155 ?( IsStrictlyLower_v<MT1> ? index : index+1UL )
1156 :( N ) );
1157 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1158
1159 for( size_t j=jbegin; j<jend; ++j ) {
1160 y[j] -= element->value() * A(index,j);
1161 }
1162 }
1163 }
1164 }
1166 //**********************************************************************************************
1167
1168 //**Optimized subtraction assignment to dense vectors*******************************************
1182 template< typename VT1 // Type of the left-hand side target vector
1183 , typename VT2 // Type of the left-hand side vector operand
1184 , typename MT1 > // Type of the right-hand side matrix operand
1185 static inline auto selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1186 -> EnableIf_t< UseOptimizedKernel_v<VT1,VT2,MT1> >
1187 {
1188 BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1189
1190 const size_t N( A.columns() );
1191
1192 auto element( x.begin() );
1193 const auto end( x.end() );
1194
1195 const size_t ipos( prevMultiple( x.nonZeros(), 4UL ) );
1196 BLAZE_INTERNAL_ASSERT( ipos <= x.nonZeros(), "Invalid end calculation" );
1197
1198 for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1199 {
1200 const size_t i1( element->index() );
1201 const VET v1( element->value() );
1202 ++element;
1203 const size_t i2( element->index() );
1204 const VET v2( element->value() );
1205 ++element;
1206 const size_t i3( element->index() );
1207 const VET v3( element->value() );
1208 ++element;
1209 const size_t i4( element->index() );
1210 const VET v4( element->value() );
1211 ++element;
1212
1213 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1214
1215 const size_t jbegin( ( IsUpper_v<MT1> )
1216 ?( IsStrictlyUpper_v<MT1> ? i+1UL : i1 )
1217 :( 0UL ) );
1218 const size_t jend( ( IsLower_v<MT1> )
1219 ?( IsStrictlyLower_v<MT1> ? i4 : i4+1UL )
1220 :( N ) );
1221 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1222
1223 for( size_t j=jbegin; j<jend; ++j ) {
1224 y[j] -= v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1225 }
1226 }
1227 for( ; element!=end; ++element )
1228 {
1229 const size_t i1( element->index() );
1230 const VET v1( element->value() );
1231
1232 const size_t jbegin( ( IsUpper_v<MT1> )
1233 ?( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 )
1234 :( 0UL ) );
1235 const size_t jend( ( IsLower_v<MT1> )
1236 ?( IsStrictlyLower_v<MT1> ? i1 : i1+1UL )
1237 :( N ) );
1238 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1239
1240 for( size_t j=jbegin; j<jend; ++j ) {
1241 y[j] -= v1 * A(i1,j);
1242 }
1243 }
1244 }
1246 //**********************************************************************************************
1247
1248 //**Vectorized subtraction assignment to dense vectors******************************************
1262 template< typename VT1 // Type of the left-hand side target vector
1263 , typename VT2 // Type of the left-hand side vector operand
1264 , typename MT1 > // Type of the right-hand side matrix operand
1265 static inline auto selectSubAssignKernel( VT1& y, const VT2& x, const MT1& A )
1266 -> EnableIf_t< UseVectorizedKernel_v<VT1,VT2,MT1> >
1267 {
1268 BLAZE_INTERNAL_ASSERT( x.nonZeros() != 0UL, "Invalid number of non-zero elements" );
1269
1270 constexpr bool remainder( !IsPadded_v<VT1> || !IsPadded_v<MT1> );
1271
1272 const size_t N( A.columns() );
1273
1274 auto element( x.begin() );
1275 const auto end( x.end() );
1276
1277 const size_t ipos( prevMultiple( x.nonZeros(), 4UL ) );
1278 BLAZE_INTERNAL_ASSERT( ipos <= x.nonZeros(), "Invalid end calculation" );
1279
1280 for( size_t i=0UL; (i+4UL)<=ipos; i+=4UL )
1281 {
1282 const size_t i1( element->index() );
1283 const VET v1( element->value() );
1284 ++element;
1285 const size_t i2( element->index() );
1286 const VET v2( element->value() );
1287 ++element;
1288 const size_t i3( element->index() );
1289 const VET v3( element->value() );
1290 ++element;
1291 const size_t i4( element->index() );
1292 const VET v4( element->value() );
1293 ++element;
1294
1295 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse vector index detected" );
1296
1297 const SIMDType xmm1( set( v1 ) );
1298 const SIMDType xmm2( set( v2 ) );
1299 const SIMDType xmm3( set( v3 ) );
1300 const SIMDType xmm4( set( v4 ) );
1301
1302 const size_t jbegin( ( IsUpper_v<MT1> )
1303 ?( prevMultiple( ( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 ), SIMDSIZE ) )
1304 :( 0UL ) );
1305 const size_t jend( ( IsLower_v<MT1> )
1306 ?( IsStrictlyLower_v<MT1> ? i4 : i4+1UL )
1307 :( N ) );
1308 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1309
1310 const size_t jpos( remainder ? prevMultiple( jend, SIMDSIZE ) : jend );
1311 BLAZE_INTERNAL_ASSERT( jpos <= jend, "Invalid end calculation" );
1312
1313 size_t j( jbegin );
1314
1315 for( ; j<jpos; j+=SIMDSIZE ) {
1316 y.store( j, y.load(j) - xmm1 * A.load(i1,j) - xmm2 * A.load(i2,j) - xmm3 * A.load(i3,j) - xmm4 * A.load(i4,j) );
1317 }
1318 for( ; remainder && j<jend; ++j ) {
1319 y[j] -= v1 * A(i1,j) + v2 * A(i2,j) + v3 * A(i3,j) + v4 * A(i4,j);
1320 }
1321 }
1322 for( ; element!=end; ++element )
1323 {
1324 const size_t i1( element->index() );
1325 const VET v1( element->value() );
1326
1327 const SIMDType xmm1( set( v1 ) );
1328
1329 const size_t jbegin( ( IsUpper_v<MT1> )
1330 ?( prevMultiple( ( IsStrictlyUpper_v<MT1> ? i1+1UL : i1 ), SIMDSIZE ) )
1331 :( 0UL ) );
1332 const size_t jend( ( IsLower_v<MT1> )
1333 ?( IsStrictlyLower_v<MT1> ? i1 : i1+1UL )
1334 :( N ) );
1335 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
1336
1337 const size_t jpos( remainder ? prevMultiple( jend, SIMDSIZE ) : jend );
1338 BLAZE_INTERNAL_ASSERT( jpos <= jend, "Invalid end calculation" );
1339
1340 size_t j( jbegin );
1341
1342 for( ; j<jpos; j+=SIMDSIZE ) {
1343 y.store( j, y.load(j) - xmm1 * A.load(i1,j) );
1344 }
1345 for( ; remainder && j<jend; ++j ) {
1346 y[j] -= v1 * A(i1,j);
1347 }
1348 }
1349 }
1351 //**********************************************************************************************
1352
1353 //**Subtraction assignment to sparse vectors****************************************************
1354 // No special implementation for the subtraction assignment to sparse vectors.
1355 //**********************************************************************************************
1356
1357 //**Multiplication assignment to dense vectors**************************************************
1369 template< typename VT2 > // Type of the target dense vector
1370 friend inline void multAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1371 {
1373
1377
1378 BLAZE_INTERNAL_ASSERT( (*lhs).size() == rhs.size(), "Invalid vector sizes" );
1379
1380 const ResultType tmp( serial( rhs ) );
1381 multAssign( *lhs, tmp );
1382 }
1383 //**********************************************************************************************
1384
1385 //**Multiplication assignment to sparse vectors*************************************************
1386 // No special implementation for the multiplication assignment to sparse vectors.
1387 //**********************************************************************************************
1388
1389 //**Division assignment to dense vectors********************************************************
1401 template< typename VT2 > // Type of the target dense vector
1402 friend inline void divAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1403 {
1405
1409
1410 BLAZE_INTERNAL_ASSERT( (*lhs).size() == rhs.size(), "Invalid vector sizes" );
1411
1412 const ResultType tmp( serial( rhs ) );
1413 divAssign( *lhs, tmp );
1414 }
1415 //**********************************************************************************************
1416
1417 //**Division assignment to sparse vectors*******************************************************
1418 // No special implementation for the division assignment to sparse vectors.
1419 //**********************************************************************************************
1420
1421 //**SMP assignment to dense vectors*************************************************************
1436 template< typename VT2 > // Type of the target dense vector
1437 friend inline auto smpAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1439 {
1441
1442 BLAZE_INTERNAL_ASSERT( (*lhs).size() == rhs.size(), "Invalid vector sizes" );
1443
1444 // Evaluation of the left-hand side sparse vector operand
1445 LT x( rhs.vec_ );
1446 if( x.nonZeros() == 0UL ) {
1447 reset( *lhs );
1448 return;
1449 }
1450
1451 // Evaluation of the right-hand side dense matrix operand
1452 RT A( rhs.mat_ );
1453
1454 // Checking the evaluated operands
1455 BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1456 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1457 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1458 BLAZE_INTERNAL_ASSERT( A.columns() == (*lhs).size() , "Invalid vector size" );
1459
1460 // Performing the sparse vector-dense matrix multiplication
1461 smpAssign( *lhs, x * A );
1462 }
1464 //**********************************************************************************************
1465
1466 //**SMP assignment to sparse vectors************************************************************
1481 template< typename VT2 > // Type of the target sparse vector
1482 friend inline auto smpAssign( SparseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1483 -> EnableIf_t< UseSMPAssign_v<VT2> >
1484 {
1486
1490
1491 BLAZE_INTERNAL_ASSERT( (*lhs).size() == rhs.size(), "Invalid vector sizes" );
1492
1493 const ResultType tmp( rhs );
1494 smpAssign( *lhs, tmp );
1495 }
1497 //**********************************************************************************************
1498
1499 //**SMP addition assignment to dense vectors****************************************************
1513 template< typename VT2 > // Type of the target dense vector
1514 friend inline auto smpAddAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1516 {
1518
1519 BLAZE_INTERNAL_ASSERT( (*lhs).size() == rhs.size(), "Invalid vector sizes" );
1520
1521 // Evaluation of the left-hand side sparse vector operand
1522 LT x( rhs.vec_ );
1523 if( x.nonZeros() == 0UL ) return;
1524
1525 // Evaluation of the right-hand side dense matrix operand
1526 RT A( rhs.mat_ );
1527
1528 // Checking the evaluated operands
1529 BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1530 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1531 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1532 BLAZE_INTERNAL_ASSERT( A.columns() == (*lhs).size() , "Invalid vector size" );
1533
1534 // Performing the sparse vector-dense matrix multiplication
1535 smpAddAssign( *lhs, x * A );
1536 }
1537 //**********************************************************************************************
1538
1539 //**SMP addition assignment to sparse vectors***************************************************
1540 // No special implementation for the SMP addition assignment to sparse vectors.
1541 //**********************************************************************************************
1542
1543 //**SMP subtraction assignment to dense vectors*************************************************
1557 template< typename VT2 > // Type of the target dense vector
1558 friend inline auto smpSubAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1560 {
1562
1563 BLAZE_INTERNAL_ASSERT( (*lhs).size() == rhs.size(), "Invalid vector sizes" );
1564
1565 // Evaluation of the left-hand side sparse vector operand
1566 LT x( rhs.vec_ );
1567 if( x.nonZeros() == 0UL ) return;
1568
1569 // Evaluation of the right-hand side dense matrix operand
1570 RT A( rhs.mat_ );
1571
1572 // Checking the evaluated operands
1573 BLAZE_INTERNAL_ASSERT( x.size() == rhs.vec_.size() , "Invalid vector size" );
1574 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.mat_.rows() , "Invalid number of rows" );
1575 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.mat_.columns(), "Invalid number of columns" );
1576 BLAZE_INTERNAL_ASSERT( A.columns() == (*lhs).size() , "Invalid vector size" );
1577
1578 // Performing the sparse vector-dense matrix multiplication
1579 smpSubAssign( *lhs, x * A );
1580 }
1581 //**********************************************************************************************
1582
1583 //**SMP subtraction assignment to sparse vectors************************************************
1584 // No special implementation for the SMP subtraction assignment to sparse vectors.
1585 //**********************************************************************************************
1586
1587 //**SMP multiplication assignment to dense vectors**********************************************
1601 template< typename VT2 > // Type of the target dense vector
1602 friend inline auto smpMultAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1604 {
1606
1610
1611 BLAZE_INTERNAL_ASSERT( (*lhs).size() == rhs.size(), "Invalid vector sizes" );
1612
1613 const ResultType tmp( rhs );
1614 smpMultAssign( *lhs, tmp );
1615 }
1616 //**********************************************************************************************
1617
1618 //**SMP multiplication assignment to sparse vectors*********************************************
1619 // No special implementation for the SMP multiplication assignment to sparse vectors.
1620 //**********************************************************************************************
1621
1622 //**SMP division assignment to dense vectors****************************************************
1636 template< typename VT2 > // Type of the target dense vector
1637 friend inline auto smpDivAssign( DenseVector<VT2,true>& lhs, const TSVecDMatMultExpr& rhs )
1639 {
1641
1645
1646 BLAZE_INTERNAL_ASSERT( (*lhs).size() == rhs.size(), "Invalid vector sizes" );
1647
1648 const ResultType tmp( rhs );
1649 smpDivAssign( *lhs, tmp );
1650 }
1651 //**********************************************************************************************
1652
1653 //**SMP division assignment to sparse vectors***************************************************
1654 // No special implementation for the SMP division assignment to sparse vectors.
1655 //**********************************************************************************************
1656
1657 //**Compile time checks*************************************************************************
1666 //**********************************************************************************************
1667};
1668//*************************************************************************************************
1669
1670
1671
1672
1673//=================================================================================================
1674//
1675// GLOBAL BINARY ARITHMETIC OPERATORS
1676//
1677//=================================================================================================
1678
1679//*************************************************************************************************
1692template< typename VT // Type of the left-hand side sparse vector
1693 , typename MT // Type of the right-hand side dense matrix
1694 , DisableIf_t< IsZero_v<VT> >* = nullptr >
1695inline const TSVecDMatMultExpr<VT,MT>
1696 tsvecdmatmult( const SparseVector<VT,true>& vec, const DenseMatrix<MT,false>& mat )
1697{
1699
1700 BLAZE_INTERNAL_ASSERT( (*vec).size() == (*mat).rows(), "Invalid vector and matrix sizes" );
1701
1702 return TSVecDMatMultExpr<VT,MT>( *vec, *mat );
1703}
1705//*************************************************************************************************
1706
1707
1708//*************************************************************************************************
1721template< typename VT // Type of the left-hand side sparse vector
1722 , typename MT // Type of the right-hand side dense matrix
1723 , EnableIf_t< IsZero_v<VT> >* = nullptr >
1724inline decltype(auto)
1725 tsvecdmatmult( const SparseVector<VT,true>& vec, const DenseMatrix<MT,false>& mat )
1726{
1728
1729 MAYBE_UNUSED( vec );
1730
1731 BLAZE_INTERNAL_ASSERT( (*vec).size() == (*mat).rows(), "Invalid vector and matrix sizes" );
1732
1733 using ReturnType = const MultTrait_t< ResultType_t<VT>, ResultType_t<MT> >;
1734
1737
1738 return ReturnType( (*mat).columns() );
1739}
1741//*************************************************************************************************
1742
1743
1744//*************************************************************************************************
1775template< typename VT // Type of the left-hand side sparse vector
1776 , typename MT > // Type of the right-hand side dense matrix
1777inline decltype(auto)
1778 operator*( const SparseVector<VT,true>& vec, const DenseMatrix<MT,false>& mat )
1779{
1781
1783
1784 if( (*vec).size() != (*mat).rows() ) {
1785 BLAZE_THROW_INVALID_ARGUMENT( "Vector and matrix sizes do not match" );
1786 }
1787
1788 return tsvecdmatmult( *vec, *mat );
1789}
1790//*************************************************************************************************
1791
1792
1793
1794
1795//=================================================================================================
1796//
1797// ISALIGNED SPECIALIZATIONS
1798//
1799//=================================================================================================
1800
1801//*************************************************************************************************
1803template< typename VT, typename MT >
1804struct IsAligned< TSVecDMatMultExpr<VT,MT> >
1805 : public IsAligned<MT>
1806{};
1808//*************************************************************************************************
1809
1810} // namespace blaze
1811
1812#endif
Header file for auxiliary alias declarations.
typename T::CompositeType CompositeType_t
Alias declaration for nested CompositeType type definitions.
Definition: Aliases.h:110
typename T::ResultType ResultType_t
Alias declaration for nested ResultType type definitions.
Definition: Aliases.h:450
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.
Definition: Aliases.h:190
typename T::TransposeType TransposeType_t
Alias declaration for nested TransposeType type definitions.
Definition: Aliases.h:550
Header file for run time assertion macros.
Header file for the blaze::checked and blaze::unchecked instances.
Header file for the EnableIf class template.
Header file for the function trace functionality.
Header file for the HasSIMDAdd type trait.
Header file for the HasSIMDMult type trait.
Header file for the If class template.
Header file for the IsAligned type trait.
Header file for the IsComputation type trait class.
Header file for the IsDiagonal type trait.
Header file for the IsExpression type trait class.
Header file for the IsLower type trait.
Header file for the IsPadded type trait.
Header file for the IsResizable type trait.
Header file for the IsSIMDCombinable type trait.
Header file for the IsStrictlyLower type trait.
Header file for the IsStrictlyUpper type trait.
Header file for the IsUpper type trait.
Deactivation of problematic macros.
Header file for the MAYBE_UNUSED function template.
Header file for the multiplication trait.
Header file for the prevMultiple shim.
Constraints on the storage order of matrix types.
Constraint on the transpose flag of vector types.
Header file for all SIMD functionality.
Constraint on the data type.
Base class for dense matrices.
Definition: DenseMatrix.h:82
Base class for N-dimensional dense vectors.
Definition: DenseVector.h:77
SIMD characteristics of data types.
Definition: SIMDTrait.h:297
Base class for sparse vectors.
Definition: SparseVector.h:72
Expression object for transpose sparse vector-dense matrix multiplications.
Definition: TSVecDMatMultExpr.h:111
LeftOperand leftOperand() const noexcept
Returns the left-hand side sparse vector operand.
Definition: TSVecDMatMultExpr.h:312
TransposeType_t< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:198
CompositeType_t< MT > MCT
Composite type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:119
TSVecDMatMultExpr(const VT &vec, const MT &mat) noexcept
Constructor for the TSVecDMatMultExpr class.
Definition: TSVecDMatMultExpr.h:241
RightOperand mat_
Right-hand side dense matrix of the multiplication expression.
Definition: TSVecDMatMultExpr.h:374
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TSVecDMatMultExpr.h:356
ReturnType operator[](size_t index) const
Subscript operator for the direct access to the vector elements.
Definition: TSVecDMatMultExpr.h:255
If_t< IsExpression_v< MT >, const MT, const MT & > RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TSVecDMatMultExpr.h:208
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TSVecDMatMultExpr.h:366
MultTrait_t< VRT, MRT > ResultType
Result type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:197
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TSVecDMatMultExpr.h:334
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TSVecDMatMultExpr.h:346
LeftOperand vec_
Left-hand side sparse vector of the multiplication expression.
Definition: TSVecDMatMultExpr.h:373
If_t< evaluateVector, const VRT, VCT > LT
Type for the assignment of the left-hand side sparse vector operand.
Definition: TSVecDMatMultExpr.h:211
ReturnType at(size_t index) const
Checked access to the vector elements.
Definition: TSVecDMatMultExpr.h:289
ElementType_t< ResultType > ElementType
Resulting element type.
Definition: TSVecDMatMultExpr.h:199
SIMDTrait_t< ElementType > SIMDType
Resulting SIMD element type.
Definition: TSVecDMatMultExpr.h:200
static constexpr size_t SIMDSIZE
The number of elements packed within a single SIMD element.
Definition: TSVecDMatMultExpr.h:232
static constexpr bool simdEnabled
Compilation switch for the expression template evaluation strategy.
Definition: TSVecDMatMultExpr.h:219
size_t size() const noexcept
Returns the current size/dimension of the vector.
Definition: TSVecDMatMultExpr.h:302
If_t< evaluateMatrix, const MRT, MCT > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: TSVecDMatMultExpr.h:214
ResultType_t< MT > MRT
Result type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:115
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TSVecDMatMultExpr.h:201
static constexpr bool evaluateMatrix
Compilation switch for the composite type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:129
static constexpr bool evaluateVector
Compilation switch for the composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:124
static constexpr bool smpAssignable
Compilation switch for the expression template assignment strategy.
Definition: TSVecDMatMultExpr.h:226
If_t< IsExpression_v< VT >, const VT, const VT & > LeftOperand
Composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:205
const ResultType CompositeType
Data type for composite expression templates.
Definition: TSVecDMatMultExpr.h:202
ElementType_t< MRT > MET
Element type of the right-hand side dense matrix expression.
Definition: TSVecDMatMultExpr.h:117
CompositeType_t< VT > VCT
Composite type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:118
RightOperand rightOperand() const noexcept
Returns the right-hand side dense matrix operand.
Definition: TSVecDMatMultExpr.h:322
ElementType_t< VRT > VET
Element type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:116
ResultType_t< VT > VRT
Result type of the left-hand side sparse vector expression.
Definition: TSVecDMatMultExpr.h:114
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Header file for the Computation base class.
Header file for the DenseVector base class.
Header file for the MatMatMultExpr base class.
Header file for the TVecMatMultExpr base class.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:137
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:812
friend void multAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Multiplication assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ...
Definition: TSVecDMatMultExpr.h:1370
friend auto smpMultAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs) -> EnableIf_t< UseSMPAssign_v< VT2 > >
SMP multiplication assignment of a transpose sparse vector-dense matrix multiplication to a dense vec...
Definition: TSVecDMatMultExpr.h:1602
friend auto smpSubAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs) -> EnableIf_t< UseSMPAssign_v< VT2 > >
SMP subtraction assignment of a transpose sparse vector-dense matrix multiplication to a dense vector...
Definition: TSVecDMatMultExpr.h:1558
friend auto smpAddAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs) -> EnableIf_t< UseSMPAssign_v< VT2 > >
SMP addition assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ...
Definition: TSVecDMatMultExpr.h:1514
friend void subAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Subtraction assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( )...
Definition: TSVecDMatMultExpr.h:1090
friend void divAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Division assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ).
Definition: TSVecDMatMultExpr.h:1402
friend void addAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs)
Addition assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ).
Definition: TSVecDMatMultExpr.h:810
friend auto smpDivAssign(DenseVector< VT2, true > &lhs, const TSVecDMatMultExpr &rhs) -> EnableIf_t< UseSMPAssign_v< VT2 > >
SMP division assignment of a transpose sparse vector-dense matrix multiplication to a dense vector ( ...
Definition: TSVecDMatMultExpr.h:1637
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition: RowMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.
Definition: RequiresEvaluation.h:81
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_TVECMATMULTEXPR(T1, T2)
Constraint on the data type.
Definition: TVecMatMultExpr.h:104
#define BLAZE_CONSTRAINT_MUST_NOT_BE_MATMATMULTEXPR_TYPE(T)
Constraint on the data type.
Definition: MatMatMultExpr.h:83
#define BLAZE_CONSTRAINT_MUST_NOT_BE_ZERO_TYPE(T)
Constraint on the data type.
Definition: Zero.h:81
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.
Definition: DenseMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_VECTOR_TYPE(T)
Constraint on the data type.
Definition: SparseVector.h:61
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_VECTOR_TYPE(T)
Constraint on the data type.
Definition: DenseVector.h:61
#define BLAZE_CONSTRAINT_MUST_BE_ROW_VECTOR_TYPE(T)
Constraint on the data type.
Definition: RowVector.h:61
#define BLAZE_CONSTRAINT_MUST_BE_ZERO_TYPE(T)
Constraint on the data type.
Definition: Zero.h:61
typename MultTrait< T1, T2 >::Type MultTrait_t
Auxiliary alias declaration for the MultTrait class template.
Definition: MultTrait.h:165
BLAZE_ALWAYS_INLINE constexpr auto prevMultiple(T1 value, T2 factor) noexcept
Rounds down an integral value to the previous multiple of a given factor.
Definition: PrevMultiple.h:68
constexpr void reset(Matrix< MT, SO > &matrix)
Resetting the given matrix.
Definition: Matrix.h:806
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:584
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:518
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.
Definition: Assert.h:101
BLAZE_ALWAYS_INLINE const EnableIf_t< IsIntegral_v< T > &&HasSize_v< T, 1UL >, If_t< IsSigned_v< T >, SIMDint8, SIMDuint8 > > set(T value) noexcept
Sets all values in the vector to the given 1-byte integral value.
Definition: Set.h:75
typename SIMDTrait< T >::Type SIMDTrait_t
Auxiliary alias declaration for the SIMDTrait class template.
Definition: SIMDTrait.h:315
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:158
typename EnableIf< Condition, T >::Type EnableIf_t
Auxiliary type for the EnableIf class template.
Definition: EnableIf.h:138
constexpr void MAYBE_UNUSED(const Args &...)
Suppression of unused parameter warnings.
Definition: MaybeUnused.h:81
typename If< Condition >::template Type< T1, T2 > If_t
Auxiliary alias template for the If class template.
Definition: If.h:108
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.
Definition: Exception.h:331
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.
Definition: Exception.h:235
#define BLAZE_FUNCTION_TRACE
Function trace macro.
Definition: FunctionTrace.h:94
constexpr Unchecked unchecked
Global Unchecked instance.
Definition: Check.h:146
Header file for the exception macros of the math module.
Header file for all forward declarations for expression class templates.
Header file for the reset shim.
Header file for the serial shim.
Base class for all compute expression templates.
Definition: Computation.h:68
Base class for all vector/matrix multiplication expression templates.
Definition: TVecMatMultExpr.h:69
System settings for performance optimizations.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
Header file for the IsZero type trait.
Header file for the RequiresEvaluation type trait.
Header file for basic type definitions.