Blaze 3.9
DMatTSMatMultExpr.h
Go to the documentation of this file.
1//=================================================================================================
33//=================================================================================================
34
35#ifndef _BLAZE_MATH_EXPRESSIONS_DMATTSMATMULTEXPR_H_
36#define _BLAZE_MATH_EXPRESSIONS_DMATTSMATMULTEXPR_H_
37
38
39//*************************************************************************************************
40// Includes
41//*************************************************************************************************
42
43#include <blaze/math/Aliases.h>
94#include <blaze/util/Assert.h>
95#include <blaze/util/EnableIf.h>
99#include <blaze/util/mpl/If.h>
100#include <blaze/util/Types.h>
103
104
105namespace blaze {
106
107//=================================================================================================
108//
109// CLASS DMATTSMATMULTEXPR
110//
111//=================================================================================================
112
113//*************************************************************************************************
120template< typename MT1 // Type of the left-hand side dense matrix
121 , typename MT2 // Type of the right-hand side sparse matrix
122 , bool SF // Symmetry flag
123 , bool HF // Hermitian flag
124 , bool LF // Lower flag
125 , bool UF > // Upper flag
127 : public MatMatMultExpr< DenseMatrix< DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, false > >
128 , private Computation
129{
130 private:
131 //**Type definitions****************************************************************************
138 //**********************************************************************************************
139
140 //**********************************************************************************************
142 static constexpr bool evaluateLeft = ( IsComputation_v<MT1> || RequiresEvaluation_v<MT1> );
143 //**********************************************************************************************
144
145 //**********************************************************************************************
147 static constexpr bool evaluateRight = ( IsComputation_v<MT2> || RequiresEvaluation_v<MT2> );
148 //**********************************************************************************************
149
150 //**********************************************************************************************
151 static constexpr bool SYM = ( SF && !( HF || LF || UF ) );
152 static constexpr bool HERM = ( HF && !( LF || UF ) );
153 static constexpr bool LOW = ( LF || ( ( SF || HF ) && UF ) );
154 static constexpr bool UPP = ( UF || ( ( SF || HF ) && LF ) );
155 //**********************************************************************************************
156
157 //**********************************************************************************************
159
164 template< typename T1, typename T2, typename T3 >
165 static constexpr bool CanExploitSymmetry_v = IsSymmetric_v<T2>;
167 //**********************************************************************************************
168
169 //**********************************************************************************************
171
175 template< typename T1, typename T2, typename T3 >
176 static constexpr bool IsEvaluationRequired_v =
177 ( ( evaluateLeft || evaluateRight ) && !CanExploitSymmetry_v<T1,T2,T3> );
179 //**********************************************************************************************
180
181 //**********************************************************************************************
183
186 template< typename T1, typename T2, typename T3 >
187 static constexpr bool UseOptimizedKernel_v =
188 ( useOptimizedKernels &&
189 !IsDiagonal_v<T2> &&
190 !IsResizable_v< ElementType_t<T1> > &&
191 !IsResizable_v<ET2> );
193 //**********************************************************************************************
194
195 //**********************************************************************************************
197
200 using ForwardFunctor = If_t< HERM
201 , DeclHerm
202 , If_t< SYM
203 , DeclSym
204 , If_t< LOW
205 , If_t< UPP
206 , DeclDiag
207 , DeclLow >
208 , If_t< UPP
209 , DeclUpp
210 , Noop > > > >;
212 //**********************************************************************************************
213
214 public:
215 //**Type definitions****************************************************************************
218
221
223 using ResultType = typename If_t< HERM
225 , If_t< SYM
227 , If_t< LOW
228 , If_t< UPP
231 , If_t< UPP
233 , MultTrait<RT1,RT2> > > > >::Type;
234
238 using ReturnType = const ElementType;
239 using CompositeType = const ResultType;
240
242 using LeftOperand = If_t< IsExpression_v<MT1>, const MT1, const MT1& >;
243
245 using RightOperand = If_t< IsExpression_v<MT2>, const MT2, const MT2& >;
246
249
252 //**********************************************************************************************
253
254 //**Compilation flags***************************************************************************
256 static constexpr bool simdEnabled = false;
257
259 static constexpr bool smpAssignable =
260 ( !evaluateLeft && MT1::smpAssignable && !evaluateRight && MT2::smpAssignable );
261 //**********************************************************************************************
262
263 //**Constructor*********************************************************************************
269 inline DMatTSMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
270 : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
271 , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
272 {
273 BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
274 }
275 //**********************************************************************************************
276
277 //**Access operator*****************************************************************************
284 inline ReturnType operator()( size_t i, size_t j ) const {
285 BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
286 BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
287
288 if( IsDiagonal_v<MT1> ) {
289 return lhs_(i,i) * rhs_(i,j);
290 }
291 else if( IsDiagonal_v<MT2> ) {
292 return lhs_(i,j) * rhs_(j,j);
293 }
294 else if( IsTriangular_v<MT1> || IsTriangular_v<MT2> ) {
295 const size_t begin( ( IsUpper_v<MT1> )
296 ?( ( IsLower_v<MT2> )
297 ?( max( ( IsStrictlyUpper_v<MT1> ? i+1UL : i )
298 , ( IsStrictlyLower_v<MT2> ? j+1UL : j ) ) )
299 :( IsStrictlyUpper_v<MT1> ? i+1UL : i ) )
300 :( ( IsLower_v<MT2> )
301 ?( IsStrictlyLower_v<MT2> ? j+1UL : j )
302 :( 0UL ) ) );
303 const size_t end( ( IsLower_v<MT1> )
304 ?( ( IsUpper_v<MT2> )
305 ?( min( ( IsStrictlyLower_v<MT1> ? i : i+1UL )
306 , ( IsStrictlyUpper_v<MT2> ? j : j+1UL ) ) )
307 :( IsStrictlyLower_v<MT1> ? i : i+1UL ) )
308 :( ( IsUpper_v<MT2> )
309 ?( IsStrictlyUpper_v<MT2> ? j : j+1UL )
310 :( lhs_.columns() ) ) );
311
312 if( begin >= end ) return ElementType();
313
314 const size_t n( end - begin );
315
316 return subvector( row( lhs_, i, unchecked ), begin, n, unchecked ) *
318 }
319 else {
320 return row( lhs_, i, unchecked ) * column( rhs_, j, unchecked );
321 }
322 }
323 //**********************************************************************************************
324
325 //**At function*********************************************************************************
333 inline ReturnType at( size_t i, size_t j ) const {
334 if( i >= lhs_.rows() ) {
335 BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
336 }
337 if( j >= rhs_.columns() ) {
338 BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
339 }
340 return (*this)(i,j);
341 }
342 //**********************************************************************************************
343
344 //**Rows function*******************************************************************************
349 inline size_t rows() const noexcept {
350 return lhs_.rows();
351 }
352 //**********************************************************************************************
353
354 //**Columns function****************************************************************************
359 inline size_t columns() const noexcept {
360 return rhs_.columns();
361 }
362 //**********************************************************************************************
363
364 //**Left operand access*************************************************************************
369 inline LeftOperand leftOperand() const noexcept {
370 return lhs_;
371 }
372 //**********************************************************************************************
373
374 //**Right operand access************************************************************************
379 inline RightOperand rightOperand() const noexcept {
380 return rhs_;
381 }
382 //**********************************************************************************************
383
384 //**********************************************************************************************
390 template< typename T >
391 inline bool canAlias( const T* alias ) const noexcept {
392 return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
393 }
394 //**********************************************************************************************
395
396 //**********************************************************************************************
402 template< typename T >
403 inline bool isAliased( const T* alias ) const noexcept {
404 return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
405 }
406 //**********************************************************************************************
407
408 //**********************************************************************************************
413 inline bool isAligned() const noexcept {
414 return lhs_.isAligned();
415 }
416 //**********************************************************************************************
417
418 //**********************************************************************************************
423 inline bool canSMPAssign() const noexcept {
424 return ( rows() * columns() >= SMP_DMATTSMATMULT_THRESHOLD ) && !IsDiagonal_v<MT1>;
425 }
426 //**********************************************************************************************
427
428 private:
429 //**Member variables****************************************************************************
432 //**********************************************************************************************
433
434 //**Assignment to dense matrices****************************************************************
447 template< typename MT // Type of the target dense matrix
448 , bool SO > // Storage order of the target dense matrix
449 friend inline auto assign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
451 {
453
454 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
455 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
456
457 LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
458 RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
459
460 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
461 BLAZE_INTERNAL_ASSERT( A.columns() == B.rows() , "Invalid matrix sizes" );
462 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns(), "Invalid number of columns" );
463
464 DMatTSMatMultExpr::selectAssignKernel( *lhs, A, B );
465 }
467 //**********************************************************************************************
468
469 //**Default assignment to dense matrices********************************************************
483 template< typename MT3 // Type of the left-hand side target matrix
484 , typename MT4 // Type of the left-hand side matrix operand
485 , typename MT5 > // Type of the right-hand side matrix operand
486 static inline auto selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
488 {
489 const size_t M( A.rows() );
490 const size_t N( B.columns() );
491
492 BLAZE_INTERNAL_ASSERT( !( SYM || HERM || LOW || UPP ) || M == N, "Broken invariant detected" );
493
494 if( LOW && UPP ) {
495 reset( C );
496 }
497
498 {
499 size_t i( 0UL );
500
501 for( ; (i+4UL) <= M; i+=4UL ) {
502 for( size_t j=( SYM || HERM || UPP ? i : 0UL ); j<( LOW ? i+4UL : N ); ++j )
503 {
504 auto element( ( IsUpper_v<MT4> )
505 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
506 :( B.begin(j) ) );
507 const auto end( ( IsLower_v<MT4> )
508 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
509 :( B.end(j) ) );
510
511 if( element == end ) {
512 reset( C(i ,j) );
513 reset( C(i+1UL,j) );
514 reset( C(i+2UL,j) );
515 reset( C(i+3UL,j) );
516 continue;
517 }
518
519 C(i ,j) = A(i ,element->index()) * element->value();
520 C(i+1UL,j) = A(i+1UL,element->index()) * element->value();
521 C(i+2UL,j) = A(i+2UL,element->index()) * element->value();
522 C(i+3UL,j) = A(i+3UL,element->index()) * element->value();
523 ++element;
524 for( ; element!=end; ++element ) {
525 C(i ,j) += A(i ,element->index()) * element->value();
526 C(i+1UL,j) += A(i+1UL,element->index()) * element->value();
527 C(i+2UL,j) += A(i+2UL,element->index()) * element->value();
528 C(i+3UL,j) += A(i+3UL,element->index()) * element->value();
529 }
530 }
531 }
532
533 for( ; (i+2UL) <= M; i+=2UL ) {
534 for( size_t j=( SYM || HERM || UPP ? i : 0UL ); j<( LOW ? i+2UL : N ); ++j )
535 {
536 auto element( ( IsUpper_v<MT4> )
537 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
538 :( B.begin(j) ) );
539 const auto end( ( IsLower_v<MT4> )
540 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
541 :( B.end(j) ) );
542
543 if( element == end ) {
544 reset( C(i ,j) );
545 reset( C(i+1UL,j) );
546 continue;
547 }
548
549 C(i ,j) = A(i ,element->index()) * element->value();
550 C(i+1UL,j) = A(i+1UL,element->index()) * element->value();
551 ++element;
552 for( ; element!=end; ++element ) {
553 C(i ,j) += A(i ,element->index()) * element->value();
554 C(i+1UL,j) += A(i+1UL,element->index()) * element->value();
555 }
556 }
557 }
558
559 for( ; i<M; ++i ) {
560 for( size_t j=( SYM || HERM || UPP ? i : 0UL ); j<( LOW ? i+1UL : N ); ++j )
561 {
562 auto element( ( IsUpper_v<MT4> )
563 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
564 :( B.begin(j) ) );
565 const auto end( ( IsLower_v<MT4> )
566 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i,j) : B.upperBound(i,j) )
567 :( B.end(j) ) );
568
569 if( element == end ) {
570 reset( C(i,j) );
571 continue;
572 }
573
574 C(i,j) = A(i,element->index()) * element->value();
575 ++element;
576 for( ; element!=end; ++element )
577 C(i,j) += A(i,element->index()) * element->value();
578 }
579 }
580 }
581
582 if( SYM || HERM ) {
583 for( size_t i=1UL; i<M; ++i ) {
584 for( size_t j=0UL; j<i; ++j ) {
585 C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
586 }
587 }
588 }
589 else if( LOW && !UPP ) {
590 for( size_t j=1UL; j<N; ++j ) {
591 for( size_t i=0UL; i<j; ++i ) {
592 reset( C(i,j) );
593 }
594 }
595 }
596 else if( !LOW && UPP ) {
597 for( size_t i=1UL; i<M; ++i ) {
598 for( size_t j=0UL; j<i; ++j ) {
599 reset( C(i,j) );
600 }
601 }
602 }
603 }
605 //**********************************************************************************************
606
607 //**Optimized assignment to dense matrices******************************************************
621 template< typename MT3 // Type of the left-hand side target matrix
622 , typename MT4 // Type of the left-hand side matrix operand
623 , typename MT5 > // Type of the right-hand side matrix operand
624 static inline auto selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
625 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
626 {
627 const size_t M( A.rows() );
628 const size_t N( B.columns() );
629
630 BLAZE_INTERNAL_ASSERT( !( SYM || HERM || LOW || UPP ) || M == N, "Broken invariant detected" );
631
632 reset( C );
633
634 {
635 size_t i( 0UL );
636
637 for( ; (i+4UL) <= M; i+=4UL ) {
638 for( size_t j=( SYM || HERM || UPP ? i : 0UL ); j<( LOW ? i+4UL : N ); ++j )
639 {
640 auto element( ( IsUpper_v<MT4> )
641 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
642 :( B.begin(j) ) );
643 const auto end( ( IsLower_v<MT4> )
644 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
645 :( B.end(j) ) );
646
647 const size_t nonzeros( end - element );
648 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
649 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
650
651 for( size_t k=0UL; k<kpos; k+=4UL )
652 {
653 const size_t j1( element->index() );
654 const ET2 v1( element->value() );
655 ++element;
656 const size_t j2( element->index() );
657 const ET2 v2( element->value() );
658 ++element;
659 const size_t j3( element->index() );
660 const ET2 v3( element->value() );
661 ++element;
662 const size_t j4( element->index() );
663 const ET2 v4( element->value() );
664 ++element;
665
666 BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
667
668 C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
669 C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
670 C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
671 C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
672 }
673
674 for( ; element!=end; ++element )
675 {
676 const size_t j1( element->index() );
677 const ET2 v1( element->value() );
678
679 C(i ,j) += A(i ,j1) * v1;
680 C(i+1UL,j) += A(i+1UL,j1) * v1;
681 C(i+2UL,j) += A(i+2UL,j1) * v1;
682 C(i+3UL,j) += A(i+3UL,j1) * v1;
683 }
684 }
685 }
686
687 for( ; (i+2UL) <= M; i+=2UL ) {
688 for( size_t j=( SYM || HERM || UPP ? i : 0UL ); j<( LOW ? i+2UL : N ); ++j )
689 {
690 auto element( ( IsUpper_v<MT4> )
691 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
692 :( B.begin(j) ) );
693 const auto end( ( IsLower_v<MT4> )
694 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
695 :( B.end(j) ) );
696
697 const size_t nonzeros( end - element );
698 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
699 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
700
701 for( size_t k=0UL; k<kpos; k+=4UL )
702 {
703 const size_t j1( element->index() );
704 const ET2 v1( element->value() );
705 ++element;
706 const size_t j2( element->index() );
707 const ET2 v2( element->value() );
708 ++element;
709 const size_t j3( element->index() );
710 const ET2 v3( element->value() );
711 ++element;
712 const size_t j4( element->index() );
713 const ET2 v4( element->value() );
714 ++element;
715
716 BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
717
718 C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
719 C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
720 }
721
722 for( ; element!=end; ++element )
723 {
724 const size_t j1( element->index() );
725 const ET2 v1( element->value() );
726
727 C(i ,j) += A(i ,j1) * v1;
728 C(i+1UL,j) += A(i+1UL,j1) * v1;
729 }
730 }
731 }
732
733 for( ; i<M; ++i ) {
734 for( size_t j=( SYM || HERM || UPP ? i : 0UL ); j<( LOW ? i+1UL : N ); ++j )
735 {
736 auto element( ( IsUpper_v<MT4> )
737 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
738 :( B.begin(j) ) );
739 const auto end( ( IsLower_v<MT4> )
740 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i,j) : B.upperBound(i,j) )
741 :( B.end(j) ) );
742
743 const size_t nonzeros( end - element );
744 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
745 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
746
747 for( size_t k=0UL; k<kpos; k+=4UL )
748 {
749 const size_t j1( element->index() );
750 const ET2 v1( element->value() );
751 ++element;
752 const size_t j2( element->index() );
753 const ET2 v2( element->value() );
754 ++element;
755 const size_t j3( element->index() );
756 const ET2 v3( element->value() );
757 ++element;
758 const size_t j4( element->index() );
759 const ET2 v4( element->value() );
760 ++element;
761
762 BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
763
764 C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
765 }
766
767 for( ; element!=end; ++element )
768 {
769 const size_t j1( element->index() );
770 const ET2 v1( element->value() );
771
772 C(i,j) += A(i,j1) * v1;
773 }
774 }
775 }
776 }
777
778 if( SYM || HERM ) {
779 for( size_t i=1UL; i<M; ++i ) {
780 for( size_t j=0UL; j<i; ++j ) {
781 C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
782 }
783 }
784 }
785 }
787 //**********************************************************************************************
788
789 //**Assignment to sparse matrices***************************************************************
802 template< typename MT // Type of the target sparse matrix
803 , bool SO > // Storage order of the target sparse matrix
804 friend inline auto assign( SparseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
805 -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
806 {
808
809 using TmpType = If_t< SO, OppositeType, ResultType >;
810
817
818 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
819 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
820
821 const ForwardFunctor fwd;
822
823 const TmpType tmp( serial( rhs ) );
824 assign( *lhs, fwd( tmp ) );
825 }
827 //**********************************************************************************************
828
829 //**Restructuring assignment********************************************************************
844 template< typename MT // Type of the target matrix
845 , bool SO > // Storage order of the target matrix
846 friend inline auto assign( Matrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
847 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
848 {
850
851 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
852 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
853
854 const ForwardFunctor fwd;
855
856 assign( *lhs, fwd( trans( rhs.lhs_ ) * rhs.rhs_ ) );
857 }
859 //**********************************************************************************************
860
861 //**Addition assignment to dense matrices*******************************************************
874 template< typename MT // Type of the target dense matrix
875 , bool SO > // Storage order of the target dense matrix
876 friend inline auto addAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
877 -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
878 {
880
881 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
882 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
883
884 LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
885 RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
886
887 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
888 BLAZE_INTERNAL_ASSERT( A.columns() == B.rows() , "Invalid matrix sizes" );
889 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns(), "Invalid number of columns" );
890
891 DMatTSMatMultExpr::selectAddAssignKernel( *lhs, A, B );
892 }
894 //**********************************************************************************************
895
896 //**Default addition assignment to dense matrices***********************************************
910 template< typename MT3 // Type of the left-hand side target matrix
911 , typename MT4 // Type of the left-hand side matrix operand
912 , typename MT5 > // Type of the right-hand side matrix operand
913 static inline auto selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
914 -> DisableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
915 {
916 const size_t M( A.rows() );
917 const size_t N( B.columns() );
918
919 BLAZE_INTERNAL_ASSERT( !( LOW || UPP ) || M == N, "Broken invariant detected" );
920
921 {
922 size_t i( 0UL );
923
924 for( ; (i+4UL) <= M; i+=4UL ) {
925 for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+4UL : N ); ++j )
926 {
927 auto element( ( IsUpper_v<MT4> )
928 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
929 :( B.begin(j) ) );
930 const auto end( ( IsLower_v<MT4> )
931 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
932 :( B.end(j) ) );
933
934 for( ; element!=end; ++element ) {
935 C(i ,j) += A(i ,element->index()) * element->value();
936 C(i+1UL,j) += A(i+1UL,element->index()) * element->value();
937 C(i+2UL,j) += A(i+2UL,element->index()) * element->value();
938 C(i+3UL,j) += A(i+3UL,element->index()) * element->value();
939 }
940 }
941 }
942
943 for( ; (i+2UL) <= M; i+=2UL ) {
944 for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+2UL : N ); ++j )
945 {
946 auto element( ( IsUpper_v<MT4> )
947 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
948 :( B.begin(j) ) );
949 const auto end( ( IsLower_v<MT4> )
950 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
951 :( B.end(j) ) );
952
953 for( ; element!=end; ++element ) {
954 C(i ,j) += A(i ,element->index()) * element->value();
955 C(i+1UL,j) += A(i+1UL,element->index()) * element->value();
956 }
957 }
958 }
959
960 for( ; i<M; ++i ) {
961 for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+1UL : N ); ++j )
962 {
963 auto element( ( IsUpper_v<MT4> )
964 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
965 :( B.begin(j) ) );
966 const auto end( ( IsLower_v<MT4> )
967 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i,j) : B.upperBound(i,j) )
968 :( B.end(j) ) );
969
970 for( ; element!=end; ++element )
971 C(i,j) += A(i,element->index()) * element->value();
972 }
973 }
974 }
975 }
977 //**********************************************************************************************
978
979 //**Optimized addition assignment to dense matrices*********************************************
993 template< typename MT3 // Type of the left-hand side target matrix
994 , typename MT4 // Type of the left-hand side matrix operand
995 , typename MT5 > // Type of the right-hand side matrix operand
996 static inline auto selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
997 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
998 {
999 const size_t M( A.rows() );
1000 const size_t N( B.columns() );
1001
1002 BLAZE_INTERNAL_ASSERT( !( LOW || UPP ) || M == N, "Broken invariant detected" );
1003
1004 {
1005 size_t i( 0UL );
1006
1007 for( ; (i+4UL) <= M; i+=4UL ) {
1008 for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+4UL : N ); ++j )
1009 {
1010 auto element( ( IsUpper_v<MT4> )
1011 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1012 :( B.begin(j) ) );
1013 const auto end( ( IsLower_v<MT4> )
1014 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
1015 :( B.end(j) ) );
1016
1017 const size_t nonzeros( end - element );
1018 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1019 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1020
1021 for( size_t k=0UL; k<kpos; k+=4UL )
1022 {
1023 const size_t j1( element->index() );
1024 const ET2 v1( element->value() );
1025 ++element;
1026 const size_t j2( element->index() );
1027 const ET2 v2( element->value() );
1028 ++element;
1029 const size_t j3( element->index() );
1030 const ET2 v3( element->value() );
1031 ++element;
1032 const size_t j4( element->index() );
1033 const ET2 v4( element->value() );
1034 ++element;
1035
1036 BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1037
1038 C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1039 C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1040 C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1041 C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1042 }
1043
1044 for( ; element!=end; ++element )
1045 {
1046 const size_t j1( element->index() );
1047 const ET2 v1( element->value() );
1048
1049 C(i ,j) += A(i ,j1) * v1;
1050 C(i+1UL,j) += A(i+1UL,j1) * v1;
1051 C(i+2UL,j) += A(i+2UL,j1) * v1;
1052 C(i+3UL,j) += A(i+3UL,j1) * v1;
1053 }
1054 }
1055 }
1056
1057 for( ; (i+2UL) <= M; i+=2UL ) {
1058 for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+2UL : N ); ++j )
1059 {
1060 auto element( ( IsUpper_v<MT4> )
1061 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1062 :( B.begin(j) ) );
1063 const auto end( ( IsLower_v<MT4> )
1064 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
1065 :( B.end(j) ) );
1066
1067 const size_t nonzeros( end - element );
1068 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1069 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1070
1071 for( size_t k=0UL; k<kpos; k+=4UL )
1072 {
1073 const size_t j1( element->index() );
1074 const ET2 v1( element->value() );
1075 ++element;
1076 const size_t j2( element->index() );
1077 const ET2 v2( element->value() );
1078 ++element;
1079 const size_t j3( element->index() );
1080 const ET2 v3( element->value() );
1081 ++element;
1082 const size_t j4( element->index() );
1083 const ET2 v4( element->value() );
1084 ++element;
1085
1086 BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1087
1088 C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1089 C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1090 }
1091
1092 for( ; element!=end; ++element )
1093 {
1094 const size_t j1( element->index() );
1095 const ET2 v1( element->value() );
1096
1097 C(i ,j) += A(i ,j1) * v1;
1098 C(i+1UL,j) += A(i+1UL,j1) * v1;
1099 }
1100 }
1101 }
1102
1103 for( ; i<M; ++i ) {
1104 for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+1UL : N ); ++j )
1105 {
1106 auto element( ( IsUpper_v<MT4> )
1107 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1108 :( B.begin(j) ) );
1109 const auto end( ( IsLower_v<MT4> )
1110 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i,j) : B.upperBound(i,j) )
1111 :( B.end(j) ) );
1112
1113 const size_t nonzeros( end - element );
1114 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1115 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1116
1117 for( size_t k=0UL; k<kpos; k+=4UL )
1118 {
1119 const size_t j1( element->index() );
1120 const ET2 v1( element->value() );
1121 ++element;
1122 const size_t j2( element->index() );
1123 const ET2 v2( element->value() );
1124 ++element;
1125 const size_t j3( element->index() );
1126 const ET2 v3( element->value() );
1127 ++element;
1128 const size_t j4( element->index() );
1129 const ET2 v4( element->value() );
1130 ++element;
1131
1132 BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1133
1134 C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1135 }
1136
1137 for( ; element!=end; ++element )
1138 {
1139 const size_t j1( element->index() );
1140 const ET2 v1( element->value() );
1141
1142 C(i,j) += A(i,j1) * v1;
1143 }
1144 }
1145 }
1146 }
1147 }
1149 //**********************************************************************************************
1150
1151 //**Restructuring addition assignment***********************************************************
1166 template< typename MT // Type of the target matrix
1167 , bool SO > // Storage order of the target matrix
1168 friend inline auto addAssign( Matrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1169 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1170 {
1172
1174
1175 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1176 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1177
1178 const ForwardFunctor fwd;
1179
1180 addAssign( *lhs, fwd( trans( rhs.lhs_ ) * rhs.rhs_ ) );
1181 }
1183 //**********************************************************************************************
1184
1185 //**Addition assignment to sparse matrices******************************************************
1186 // No special implementation for the addition assignment to sparse matrices.
1187 //**********************************************************************************************
1188
1189 //**Subtraction assignment to dense matrices****************************************************
1202 template< typename MT // Type of the target dense matrix
1203 , bool SO > // Storage order of the target dense matrix
1204 friend inline auto subAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1205 -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1206 {
1208
1209 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1210 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1211
1212 LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1213 RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1214
1215 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1216 BLAZE_INTERNAL_ASSERT( A.columns() == B.rows() , "Invalid matrix sizes" );
1217 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns(), "Invalid number of columns" );
1218
1219 DMatTSMatMultExpr::selectSubAssignKernel( *lhs, A, B );
1220 }
1222 //**********************************************************************************************
1223
1224 //**Default subtraction assignment to dense matrices********************************************
1238 template< typename MT3 // Type of the left-hand side target matrix
1239 , typename MT4 // Type of the left-hand side matrix operand
1240 , typename MT5 > // Type of the right-hand side matrix operand
1241 static inline auto selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1242 -> DisableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1243 {
1244 const size_t M( A.rows() );
1245 const size_t N( B.columns() );
1246
1247 BLAZE_INTERNAL_ASSERT( !( LOW || UPP ) || M == N, "Broken invariant detected" );
1248
1249 {
1250 size_t i( 0UL );
1251
1252 for( ; (i+4UL) <= M; i+=4UL ) {
1253 for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+4UL : N ); ++j )
1254 {
1255 auto element( ( IsUpper_v<MT4> )
1256 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1257 :( B.begin(j) ) );
1258 const auto end( ( IsLower_v<MT4> )
1259 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
1260 :( B.end(j) ) );
1261
1262 for( ; element!=end; ++element ) {
1263 C(i ,j) -= A(i ,element->index()) * element->value();
1264 C(i+1UL,j) -= A(i+1UL,element->index()) * element->value();
1265 C(i+2UL,j) -= A(i+2UL,element->index()) * element->value();
1266 C(i+3UL,j) -= A(i+3UL,element->index()) * element->value();
1267 }
1268 }
1269 }
1270
1271 for( ; (i+2UL) <= M; i+=2UL ) {
1272 for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+2UL : N ); ++j )
1273 {
1274 auto element( ( IsUpper_v<MT4> )
1275 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1276 :( B.begin(j) ) );
1277 const auto end( ( IsLower_v<MT4> )
1278 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
1279 :( B.end(j) ) );
1280
1281 for( ; element!=end; ++element ) {
1282 C(i ,j) -= A(i ,element->index()) * element->value();
1283 C(i+1UL,j) -= A(i+1UL,element->index()) * element->value();
1284 }
1285 }
1286 }
1287
1288 for( ; i<M; ++i ) {
1289 for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+1UL : N ); ++j )
1290 {
1291 auto element( ( IsUpper_v<MT4> )
1292 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1293 :( B.begin(j) ) );
1294 const auto end( ( IsLower_v<MT4> )
1295 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i,j) : B.upperBound(i,j) )
1296 :( B.end(j) ) );
1297
1298 for( ; element!=end; ++element )
1299 C(i,j) -= A(i,element->index()) * element->value();
1300 }
1301 }
1302 }
1303 }
1305 //**********************************************************************************************
1306
1307 //**Optimized subtraction assignment to dense matrices******************************************
1321 template< typename MT3 // Type of the left-hand side target matrix
1322 , typename MT4 // Type of the left-hand side matrix operand
1323 , typename MT5 > // Type of the right-hand side matrix operand
1324 static inline auto selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1325 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1326 {
1327 const size_t M( A.rows() );
1328 const size_t N( B.columns() );
1329
1330 BLAZE_INTERNAL_ASSERT( !( LOW || UPP ) || M == N, "Broken invariant detected" );
1331
1332 {
1333 size_t i( 0UL );
1334
1335 for( ; (i+4UL) <= M; i+=4UL ) {
1336 for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+4UL : N ); ++j )
1337 {
1338 auto element( ( IsUpper_v<MT4> )
1339 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1340 :( B.begin(j) ) );
1341 const auto end( ( IsLower_v<MT4> )
1342 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
1343 :( B.end(j) ) );
1344
1345 const size_t nonzeros( end - element );
1346 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1347 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1348
1349 for( size_t k=0UL; k<kpos; k+=4UL )
1350 {
1351 const size_t j1( element->index() );
1352 const ET2 v1( element->value() );
1353 ++element;
1354 const size_t j2( element->index() );
1355 const ET2 v2( element->value() );
1356 ++element;
1357 const size_t j3( element->index() );
1358 const ET2 v3( element->value() );
1359 ++element;
1360 const size_t j4( element->index() );
1361 const ET2 v4( element->value() );
1362 ++element;
1363
1364 BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1365
1366 C(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1367 C(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1368 C(i+2UL,j) -= A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1369 C(i+3UL,j) -= A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1370 }
1371
1372 for( ; element!=end; ++element )
1373 {
1374 const size_t j1( element->index() );
1375 const ET2 v1( element->value() );
1376
1377 C(i ,j) -= A(i ,j1) * v1;
1378 C(i+1UL,j) -= A(i+1UL,j1) * v1;
1379 C(i+2UL,j) -= A(i+2UL,j1) * v1;
1380 C(i+3UL,j) -= A(i+3UL,j1) * v1;
1381 }
1382 }
1383 }
1384
1385 for( ; (i+2UL) <= M; i+=2UL ) {
1386 for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+2UL : N ); ++j )
1387 {
1388 auto element( ( IsUpper_v<MT4> )
1389 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1390 :( B.begin(j) ) );
1391 const auto end( ( IsLower_v<MT4> )
1392 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
1393 :( B.end(j) ) );
1394
1395 const size_t nonzeros( end - element );
1396 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1397 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1398
1399 for( size_t k=0UL; k<kpos; k+=4UL )
1400 {
1401 const size_t j1( element->index() );
1402 const ET2 v1( element->value() );
1403 ++element;
1404 const size_t j2( element->index() );
1405 const ET2 v2( element->value() );
1406 ++element;
1407 const size_t j3( element->index() );
1408 const ET2 v3( element->value() );
1409 ++element;
1410 const size_t j4( element->index() );
1411 const ET2 v4( element->value() );
1412 ++element;
1413
1414 BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1415
1416 C(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1417 C(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1418 }
1419
1420 for( ; element!=end; ++element )
1421 {
1422 const size_t j1( element->index() );
1423 const ET2 v1( element->value() );
1424
1425 C(i ,j) -= A(i ,j1) * v1;
1426 C(i+1UL,j) -= A(i+1UL,j1) * v1;
1427 }
1428 }
1429 }
1430
1431 for( ; i<M; ++i ) {
1432 for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+1UL : N ); ++j )
1433 {
1434 auto element( ( IsUpper_v<MT4> )
1435 ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1436 :( B.begin(j) ) );
1437 const auto end( ( IsLower_v<MT4> )
1438 ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i,j) : B.upperBound(i,j) )
1439 :( B.end(j) ) );
1440
1441 const size_t nonzeros( end - element );
1442 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1443 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1444
1445 for( size_t k=0UL; k<kpos; k+=4UL )
1446 {
1447 const size_t j1( element->index() );
1448 const ET2 v1( element->value() );
1449 ++element;
1450 const size_t j2( element->index() );
1451 const ET2 v2( element->value() );
1452 ++element;
1453 const size_t j3( element->index() );
1454 const ET2 v3( element->value() );
1455 ++element;
1456 const size_t j4( element->index() );
1457 const ET2 v4( element->value() );
1458 ++element;
1459
1460 BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1461
1462 C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1463 }
1464
1465 for( ; element!=end; ++element )
1466 {
1467 const size_t j1( element->index() );
1468 const ET2 v1( element->value() );
1469
1470 C(i,j) -= A(i,j1) * v1;
1471 }
1472 }
1473 }
1474 }
1475 }
1477 //**********************************************************************************************
1478
1479 //**Restructuring subtraction assignment********************************************************
1494 template< typename MT // Type of the target matrix
1495 , bool SO > // Storage order of the target matrix
1496 friend inline auto subAssign( Matrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1497 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1498 {
1500
1502
1503 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1504 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1505
1506 const ForwardFunctor fwd;
1507
1508 subAssign( *lhs, fwd( trans( rhs.lhs_ ) * rhs.rhs_ ) );
1509 }
1511 //**********************************************************************************************
1512
1513 //**Subtraction assignment to sparse matrices***************************************************
1514 // No special implementation for the subtraction assignment to sparse matrices.
1515 //**********************************************************************************************
1516
1517 //**Schur product assignment to dense matrices**************************************************
1530 template< typename MT // Type of the target dense matrix
1531 , bool SO > // Storage order of the target dense matrix
1532 friend inline void schurAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1533 {
1535
1539
1540 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1541 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1542
1543 const ResultType tmp( serial( rhs ) );
1544 schurAssign( *lhs, tmp );
1545 }
1547 //**********************************************************************************************
1548
1549 //**Schur product assignment to sparse matrices*************************************************
1550 // No special implementation for the Schur product assignment to sparse matrices.
1551 //**********************************************************************************************
1552
1553 //**Multiplication assignment to dense matrices*************************************************
1554 // No special implementation for the multiplication assignment to dense matrices.
1555 //**********************************************************************************************
1556
1557 //**Multiplication assignment to sparse matrices************************************************
1558 // No special implementation for the multiplication assignment to sparse matrices.
1559 //**********************************************************************************************
1560
1561 //**SMP assignment to dense matrices************************************************************
1576 template< typename MT // Type of the target dense matrix
1577 , bool SO > // Storage order of the target dense matrix
1578 friend inline auto smpAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1579 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1580 {
1582
1583 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1584 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1585
1586 LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1587 RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1588
1589 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1590 BLAZE_INTERNAL_ASSERT( A.columns() == B.rows() , "Invalid matrix sizes" );
1591 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns(), "Invalid number of columns" );
1592
1593 smpAssign( *lhs, A * B );
1594 }
1596 //**********************************************************************************************
1597
1598 //**SMP assignment to sparse matrices***********************************************************
1613 template< typename MT // Type of the target sparse matrix
1614 , bool SO > // Storage order of the target sparse matrix
1615 friend inline auto smpAssign( SparseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1616 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1617 {
1619
1620 using TmpType = If_t< SO, OppositeType, ResultType >;
1621
1628
1629 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1630 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1631
1632 const ForwardFunctor fwd;
1633
1634 const TmpType tmp( rhs );
1635 smpAssign( *lhs, fwd( tmp ) );
1636 }
1638 //**********************************************************************************************
1639
1640 //**Restructuring SMP assignment****************************************************************
1655 template< typename MT // Type of the target matrix
1656 , bool SO > // Storage order of the target matrix
1657 friend inline auto smpAssign( Matrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1658 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1659 {
1661
1663
1664 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1665 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1666
1667 const ForwardFunctor fwd;
1668
1669 smpAssign( *lhs, fwd( trans( rhs.lhs_ ) * rhs.rhs_ ) );
1670 }
1672 //**********************************************************************************************
1673
1674 //**SMP addition assignment to dense matrices***************************************************
1690 template< typename MT // Type of the target dense matrix
1691 , bool SO > // Storage order of the target dense matrix
1692 friend inline auto smpAddAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1693 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1694 {
1696
1697 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1698 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1699
1700 LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1701 RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1702
1703 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1704 BLAZE_INTERNAL_ASSERT( A.columns() == B.rows() , "Invalid matrix sizes" );
1705 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns(), "Invalid number of columns" );
1706
1707 smpAddAssign( *lhs, A * B );
1708 }
1710 //**********************************************************************************************
1711
1712 //**Restructuring SMP addition assignment*******************************************************
1727 template< typename MT // Type of the target matrix
1728 , bool SO > // Storage order of the target matrix
1729 friend inline auto smpAddAssign( Matrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1730 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1731 {
1733
1735
1736 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1737 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1738
1739 const ForwardFunctor fwd;
1740
1741 smpAddAssign( *lhs, fwd( trans( rhs.lhs_ ) * rhs.rhs_ ) );
1742 }
1744 //**********************************************************************************************
1745
1746 //**SMP addition assignment to sparse matrices**************************************************
1747 // No special implementation for the SMP addition assignment to sparse matrices.
1748 //**********************************************************************************************
1749
1750 //**SMP subtraction assignment to dense matrices************************************************
1766 template< typename MT // Type of the target dense matrix
1767 , bool SO > // Storage order of the target dense matrix
1768 friend inline auto smpSubAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1769 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1770 {
1772
1773 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1774 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1775
1776 LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1777 RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1778
1779 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1780 BLAZE_INTERNAL_ASSERT( A.columns() == B.rows() , "Invalid matrix sizes" );
1781 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns(), "Invalid number of columns" );
1782
1783 smpSubAssign( *lhs, A * B );
1784 }
1786 //**********************************************************************************************
1787
1788 //**Restructuring SMP subtraction assignment****************************************************
1803 template< typename MT // Type of the target matrix
1804 , bool SO > // Storage order of the target matrix
1805 friend inline auto smpSubAssign( Matrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1806 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1807 {
1809
1811
1812 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1813 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1814
1815 const ForwardFunctor fwd;
1816
1817 smpSubAssign( *lhs, fwd( trans( rhs.lhs_ ) * rhs.rhs_ ) );
1818 }
1820 //**********************************************************************************************
1821
1822 //**SMP subtraction assignment to sparse matrices***********************************************
1823 // No special implementation for the SMP subtraction assignment to sparse matrices.
1824 //**********************************************************************************************
1825
1826 //**SMP Schur product assignment to dense matrices**********************************************
1839 template< typename MT // Type of the target dense matrix
1840 , bool SO > // Storage order of the target dense matrix
1841 friend inline void smpSchurAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1842 {
1844
1848
1849 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1850 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1851
1852 const ResultType tmp( rhs );
1853 smpSchurAssign( *lhs, tmp );
1854 }
1856 //**********************************************************************************************
1857
1858 //**SMP Schur product assignment to sparse matrices*********************************************
1859 // No special implementation for the SMP Schur product assignment to sparse matrices.
1860 //**********************************************************************************************
1861
1862 //**SMP multiplication assignment to dense matrices*********************************************
1863 // No special implementation for the SMP multiplication assignment to dense matrices.
1864 //**********************************************************************************************
1865
1866 //**SMP multiplication assignment to sparse matrices********************************************
1867 // No special implementation for the SMP multiplication assignment to sparse matrices.
1868 //**********************************************************************************************
1869
1870 //**Compile time checks*************************************************************************
1879 //**********************************************************************************************
1880};
1881//*************************************************************************************************
1882
1883
1884
1885
1886//=================================================================================================
1887//
1888// GLOBAL BINARY ARITHMETIC OPERATORS
1889//
1890//=================================================================================================
1891
1892//*************************************************************************************************
1905template< typename MT1 // Type of the left-hand side dense matrix
1906 , typename MT2 // Type of the right-hand side sparse matrix
1907 , DisableIf_t< ( IsIdentity_v<MT2> &&
1908 IsSame_v< ElementType_t<MT1>, ElementType_t<MT2> > ) ||
1909 IsZero_v<MT2> >* = nullptr >
1910inline const DMatTSMatMultExpr<MT1,MT2,false,false,false,false>
1911 dmattsmatmult( const DenseMatrix<MT1,false>& lhs, const SparseMatrix<MT2,true>& rhs )
1912{
1914
1915 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == (*rhs).rows(), "Invalid matrix sizes" );
1916
1917 return DMatTSMatMultExpr<MT1,MT2,false,false,false,false>( *lhs, *rhs );
1918}
1920//*************************************************************************************************
1921
1922
1923//*************************************************************************************************
1937template< typename MT1 // Type of the left-hand side dense matrix
1938 , typename MT2 // Type of the right-hand side sparse matrix
1939 , EnableIf_t< IsIdentity_v<MT2> &&
1940 IsSame_v< ElementType_t<MT1>, ElementType_t<MT2> > >* = nullptr >
1941inline const MT1&
1942 dmattsmatmult( const DenseMatrix<MT1,false>& lhs, const SparseMatrix<MT2,true>& rhs )
1943{
1945
1946 MAYBE_UNUSED( rhs );
1947
1948 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == (*rhs).rows(), "Invalid matrix sizes" );
1949
1950 return (*lhs);
1951}
1953//*************************************************************************************************
1954
1955
1956//*************************************************************************************************
1969template< typename MT1 // Type of the left-hand side dense matrix
1970 , typename MT2 // Type of the right-hand side sparse matrix
1971 , EnableIf_t< IsZero_v<MT2> >* = nullptr >
1972inline decltype(auto)
1973 dmattsmatmult( const DenseMatrix<MT1,false>& lhs, const SparseMatrix<MT2,true>& rhs )
1974{
1976
1977 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == (*rhs).rows(), "Invalid matrix sizes" );
1978
1979 using ReturnType = const MultTrait_t< ResultType_t<MT1>, ResultType_t<MT2> >;
1980
1983
1984 return ReturnType( (*lhs).rows(), (*rhs).columns() );
1985}
1987//*************************************************************************************************
1988
1989
1990//*************************************************************************************************
2020template< typename MT1 // Type of the left-hand side dense matrix
2021 , typename MT2 > // Type of the right-hand side sparse matrix
2022inline decltype(auto)
2023 operator*( const DenseMatrix<MT1,false>& lhs, const SparseMatrix<MT2,true>& rhs )
2024{
2026
2027 if( (*lhs).columns() != (*rhs).rows() ) {
2028 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
2029 }
2030
2031 return dmattsmatmult( *lhs, *rhs );
2032}
2033//*************************************************************************************************
2034
2035
2036
2037
2038//=================================================================================================
2039//
2040// GLOBAL FUNCTIONS
2041//
2042//=================================================================================================
2043
2044//*************************************************************************************************
2068template< typename MT1 // Type of the left-hand side dense matrix
2069 , typename MT2 // Type of the right-hand side sparse matrix
2070 , bool SF // Symmetry flag
2071 , bool HF // Hermitian flag
2072 , bool LF // Lower flag
2073 , bool UF > // Upper flag
2074inline decltype(auto) declsym( const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2075{
2077
2078 if( !isSquare( dm ) ) {
2079 BLAZE_THROW_INVALID_ARGUMENT( "Invalid symmetric matrix specification" );
2080 }
2081
2082 using ReturnType = const DMatTSMatMultExpr<MT1,MT2,true,HF,LF,UF>;
2083 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2084}
2086//*************************************************************************************************
2087
2088
2089//*************************************************************************************************
2113template< typename MT1 // Type of the left-hand side dense matrix
2114 , typename MT2 // Type of the right-hand side sparse matrix
2115 , bool SF // Symmetry flag
2116 , bool HF // Hermitian flag
2117 , bool LF // Lower flag
2118 , bool UF > // Upper flag
2119inline decltype(auto) declherm( const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2120{
2122
2123 if( !isSquare( dm ) ) {
2124 BLAZE_THROW_INVALID_ARGUMENT( "Invalid Hermitian matrix specification" );
2125 }
2126
2127 using ReturnType = const DMatTSMatMultExpr<MT1,MT2,SF,true,LF,UF>;
2128 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2129}
2131//*************************************************************************************************
2132
2133
2134//*************************************************************************************************
2158template< typename MT1 // Type of the left-hand side dense matrix
2159 , typename MT2 // Type of the right-hand side sparse matrix
2160 , bool SF // Symmetry flag
2161 , bool HF // Hermitian flag
2162 , bool LF // Lower flag
2163 , bool UF > // Upper flag
2164inline decltype(auto) decllow( const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2165{
2167
2168 if( !isSquare( dm ) ) {
2169 BLAZE_THROW_INVALID_ARGUMENT( "Invalid lower matrix specification" );
2170 }
2171
2172 using ReturnType = const DMatTSMatMultExpr<MT1,MT2,SF,HF,true,UF>;
2173 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2174}
2176//*************************************************************************************************
2177
2178
2179//*************************************************************************************************
2203template< typename MT1 // Type of the left-hand side dense matrix
2204 , typename MT2 // Type of the right-hand side sparse matrix
2205 , bool SF // Symmetry flag
2206 , bool HF // Hermitian flag
2207 , bool UF > // Upper flag
2208inline decltype(auto) declunilow( const DMatTSMatMultExpr<MT1,MT2,SF,HF,false,UF>& dm )
2209{
2211
2212 if( !isSquare( dm ) ) {
2213 BLAZE_THROW_INVALID_ARGUMENT( "Invalid unilower matrix specification" );
2214 }
2215
2216 return declunilow( decllow( *dm ) );
2217}
2219//*************************************************************************************************
2220
2221
2222//*************************************************************************************************
2246template< typename MT1 // Type of the left-hand side dense matrix
2247 , typename MT2 // Type of the right-hand side sparse matrix
2248 , bool SF // Symmetry flag
2249 , bool HF // Hermitian flag
2250 , bool UF > // Upper flag
2251inline decltype(auto) declstrlow( const DMatTSMatMultExpr<MT1,MT2,SF,HF,false,UF>& dm )
2252{
2254
2255 if( !isSquare( dm ) ) {
2256 BLAZE_THROW_INVALID_ARGUMENT( "Invalid strictly lower matrix specification" );
2257 }
2258
2259 return declstrlow( decllow( *dm ) );
2260}
2262//*************************************************************************************************
2263
2264
2265//*************************************************************************************************
2289template< typename MT1 // Type of the left-hand side dense matrix
2290 , typename MT2 // Type of the right-hand side sparse matrix
2291 , bool SF // Symmetry flag
2292 , bool HF // Hermitian flag
2293 , bool LF // Lower flag
2294 , bool UF > // Upper flag
2295inline decltype(auto) declupp( const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2296{
2298
2299 if( !isSquare( dm ) ) {
2300 BLAZE_THROW_INVALID_ARGUMENT( "Invalid upper matrix specification" );
2301 }
2302
2303 using ReturnType = const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,true>;
2304 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2305}
2307//*************************************************************************************************
2308
2309
2310//*************************************************************************************************
2334template< typename MT1 // Type of the left-hand side dense matrix
2335 , typename MT2 // Type of the right-hand side sparse matrix
2336 , bool SF // Symmetry flag
2337 , bool HF // Hermitian flag
2338 , bool LF > // Lower flag
2339inline decltype(auto) decluniupp( const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,false>& dm )
2340{
2342
2343 if( !isSquare( dm ) ) {
2344 BLAZE_THROW_INVALID_ARGUMENT( "Invalid uniupper matrix specification" );
2345 }
2346
2347 return decluniupp( declupp( *dm ) );
2348}
2350//*************************************************************************************************
2351
2352
2353//*************************************************************************************************
2377template< typename MT1 // Type of the left-hand side dense matrix
2378 , typename MT2 // Type of the right-hand side sparse matrix
2379 , bool SF // Symmetry flag
2380 , bool HF // Hermitian flag
2381 , bool LF > // Lower flag
2382inline decltype(auto) declstrupp( const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,false>& dm )
2383{
2385
2386 if( !isSquare( dm ) ) {
2387 BLAZE_THROW_INVALID_ARGUMENT( "Invalid strictly upper matrix specification" );
2388 }
2389
2390 return declstrupp( declupp( *dm ) );
2391}
2393//*************************************************************************************************
2394
2395
2396//*************************************************************************************************
2420template< typename MT1 // Type of the left-hand side dense matrix
2421 , typename MT2 // Type of the right-hand side sparse matrix
2422 , bool SF // Symmetry flag
2423 , bool HF // Hermitian flag
2424 , bool LF // Lower flag
2425 , bool UF > // Upper flag
2426inline decltype(auto) decldiag( const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2427{
2429
2430 if( !isSquare( dm ) ) {
2431 BLAZE_THROW_INVALID_ARGUMENT( "Invalid diagonal matrix specification" );
2432 }
2433
2434 using ReturnType = const DMatTSMatMultExpr<MT1,MT2,SF,HF,true,true>;
2435 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2436}
2438//*************************************************************************************************
2439
2440
2441
2442
2443//=================================================================================================
2444//
2445// SIZE SPECIALIZATIONS
2446//
2447//=================================================================================================
2448
2449//*************************************************************************************************
2451template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2452struct Size< DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 0UL >
2453 : public Size<MT1,0UL>
2454{};
2455
2456template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2457struct Size< DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 1UL >
2458 : public Size<MT2,1UL>
2459{};
2461//*************************************************************************************************
2462
2463
2464
2465
2466//=================================================================================================
2467//
2468// ISALIGNED SPECIALIZATIONS
2469//
2470//=================================================================================================
2471
2472//*************************************************************************************************
2474template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2475struct IsAligned< DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2476 : public IsAligned<MT1>
2477{};
2479//*************************************************************************************************
2480
2481} // namespace blaze
2482
2483#endif
Header file for auxiliary alias declarations.
typename T::CompositeType CompositeType_t
Alias declaration for nested CompositeType type definitions.
Definition: Aliases.h:110
typename T::ResultType ResultType_t
Alias declaration for nested ResultType type definitions.
Definition: Aliases.h:450
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.
Definition: Aliases.h:190
typename T::OppositeType OppositeType_t
Alias declaration for nested OppositeType type definitions.
Definition: Aliases.h:310
typename T::TransposeType TransposeType_t
Alias declaration for nested TransposeType type definitions.
Definition: Aliases.h:550
Header file for run time assertion macros.
Header file for the blaze::checked and blaze::unchecked instances.
Constraints on the storage order of matrix types.
Header file for the conjugate shim.
Header file for the decldiag trait.
Header file for the DeclDiag functor.
Header file for the declherm trait.
Header file for the DeclHerm functor.
Header file for the decllow trait.
Header file for the DeclLow functor.
Header file for the declsym trait.
Header file for the DeclSym functor.
Header file for the declupp trait.
Header file for the DeclUpp functor.
Header file for the EnableIf class template.
Header file for the function trace functionality.
Header file for the If class template.
Header file for the IntegralConstant class template.
Header file for the IsAligned type trait.
Header file for the IsBuiltin type trait.
Header file for the IsComputation type trait class.
Header file for the IsDiagonal type trait.
Header file for the IsExpression type trait class.
Header file for the IsIdentity type trait.
Header file for the IsLower type trait.
Header file for the IsResizable type trait.
Header file for the IsSame and IsStrictlySame type traits.
Header file for the IsStrictlyLower type trait.
Header file for the IsStrictlyUpper type trait.
Header file for the IsSymmetric type trait.
Header file for the IsTriangular type trait.
Header file for the IsUpper type trait.
Header file for the MAYBE_UNUSED function template.
Header file for the multiplication trait.
Header file for the Noop functor.
Header file for the prevMultiple shim.
Constraints on the storage order of matrix types.
Constraint on the data type.
Constraint on the data type.
Expression object for dense matrix-transpose sparse matrix multiplications.
Definition: DMatTSMatMultExpr.h:129
typename If_t< HERM, DeclHermTrait< MultTrait_t< RT1, RT2 > >, If_t< SYM, DeclSymTrait< MultTrait_t< RT1, RT2 > >, If_t< LOW, If_t< UPP, DeclDiagTrait< MultTrait_t< RT1, RT2 > >, DeclLowTrait< MultTrait_t< RT1, RT2 > > >, If_t< UPP, DeclUppTrait< MultTrait_t< RT1, RT2 > >, MultTrait< RT1, RT2 > > > > >::Type ResultType
Result type for expression template evaluations.
Definition: DMatTSMatMultExpr.h:233
static constexpr bool smpAssignable
Compilation switch for the expression template assignment strategy.
Definition: DMatTSMatMultExpr.h:259
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: DMatTSMatMultExpr.h:403
const ElementType ReturnType
Return type for expression template evaluations.
Definition: DMatTSMatMultExpr.h:238
If_t< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: DMatTSMatMultExpr.h:251
ElementType_t< RT1 > ET1
Element type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:134
LeftOperand leftOperand() const noexcept
Returns the left-hand side dense matrix operand.
Definition: DMatTSMatMultExpr.h:369
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: DMatTSMatMultExpr.h:349
ElementType_t< RT2 > ET2
Element type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:135
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: DMatTSMatMultExpr.h:430
If_t< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: DMatTSMatMultExpr.h:248
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: DMatTSMatMultExpr.h:284
If_t< IsExpression_v< MT2 >, const MT2, const MT2 & > RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:245
DMatTSMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the DMatTSMatMultExpr class.
Definition: DMatTSMatMultExpr.h:269
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: DMatTSMatMultExpr.h:359
ResultType_t< MT2 > RT2
Result type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:133
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: DMatTSMatMultExpr.h:333
RightOperand rightOperand() const noexcept
Returns the right-hand side transpose sparse matrix operand.
Definition: DMatTSMatMultExpr.h:379
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: DMatTSMatMultExpr.h:423
TransposeType_t< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: DMatTSMatMultExpr.h:236
static constexpr bool evaluateLeft
Compilation switch for the composite type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:142
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: DMatTSMatMultExpr.h:431
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: DMatTSMatMultExpr.h:413
static constexpr bool UPP
Flag for upper matrices.
Definition: DMatTSMatMultExpr.h:154
static constexpr bool LOW
Flag for lower matrices.
Definition: DMatTSMatMultExpr.h:153
static constexpr bool SYM
Flag for symmetric matrices.
Definition: DMatTSMatMultExpr.h:151
static constexpr bool HERM
Flag for Hermitian matrices.
Definition: DMatTSMatMultExpr.h:152
const ResultType CompositeType
Data type for composite expression templates.
Definition: DMatTSMatMultExpr.h:239
OppositeType_t< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: DMatTSMatMultExpr.h:235
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: DMatTSMatMultExpr.h:391
CompositeType_t< MT1 > CT1
Composite type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:136
ResultType_t< MT1 > RT1
Result type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:132
ElementType_t< ResultType > ElementType
Resulting element type.
Definition: DMatTSMatMultExpr.h:237
static constexpr bool simdEnabled
Compilation switch for the expression template evaluation strategy.
Definition: DMatTSMatMultExpr.h:256
static constexpr bool evaluateRight
Compilation switch for the composite type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:147
If_t< IsExpression_v< MT1 >, const MT1, const MT1 & > LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:242
CompositeType_t< MT2 > CT2
Composite type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:137
Base class for dense matrices.
Definition: DenseMatrix.h:82
Base class for sparse matrices.
Definition: SparseMatrix.h:77
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Header file for the Computation base class.
Header file for the DenseMatrix base class.
Header file for the MatMatMultExpr base class.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:137
decltype(auto) min(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise minimum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1339
decltype(auto) max(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise maximum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1375
decltype(auto) declstrupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as strictly upper.
Definition: DMatDeclStrUppExpr.h:1003
decltype(auto) conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition: DMatMapExpr.h:1464
decltype(auto) decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as diagonal.
Definition: DMatDeclDiagExpr.h:978
decltype(auto) trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:766
decltype(auto) declstrlow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as strictly lower.
Definition: DMatDeclStrLowExpr.h:1003
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:812
decltype(auto) declupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as upper.
Definition: DMatDeclUppExpr.h:1004
decltype(auto) decllow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as lower.
Definition: DMatDeclLowExpr.h:1004
decltype(auto) decluniupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as uniupper.
Definition: DMatDeclUniUppExpr.h:1005
decltype(auto) declherm(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as Hermitian.
Definition: DMatDeclHermExpr.h:1005
decltype(auto) declsym(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as symmetric.
Definition: DMatDeclSymExpr.h:1005
decltype(auto) declunilow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as unilower.
Definition: DMatDeclUniLowExpr.h:1004
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.
Definition: Symmetric.h:79
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition: RowMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.
Definition: StorageOrder.h:84
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.
Definition: RequiresEvaluation.h:81
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.
Definition: MatMatMultExpr.h:103
#define BLAZE_CONSTRAINT_MUST_NOT_BE_ZERO_TYPE(T)
Constraint on the data type.
Definition: Zero.h:81
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.
Definition: DenseMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.
Definition: SparseMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition: ColumnMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_ZERO_TYPE(T)
Constraint on the data type.
Definition: Zero.h:61
BLAZE_ALWAYS_INLINE constexpr auto prevMultiple(T1 value, T2 factor) noexcept
Rounds down an integral value to the previous multiple of a given factor.
Definition: PrevMultiple.h:68
constexpr void reset(Matrix< MT, SO > &matrix)
Resetting the given matrix.
Definition: Matrix.h:806
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:584
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:518
bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition: Matrix.h:1383
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:137
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.
Definition: Assert.h:101
auto smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:162
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
auto smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:194
auto smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:131
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:158
constexpr void MAYBE_UNUSED(const Args &...)
Suppression of unused parameter warnings.
Definition: MaybeUnused.h:81
typename If< Condition >::template Type< T1, T2 > If_t
Auxiliary alias template for the If class template.
Definition: If.h:108
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.
Definition: Exception.h:331
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.
Definition: Exception.h:235
typename EnableIf<!Condition, T >::Type DisableIf_t
Auxiliary type for the EnableIf class template.
Definition: EnableIf.h:175
#define BLAZE_FUNCTION_TRACE
Function trace macro.
Definition: FunctionTrace.h:94
constexpr Unchecked unchecked
Global Unchecked instance.
Definition: Check.h:146
Header file for the exception macros of the math module.
Constraints on the storage order of matrix types.
Header file for all forward declarations for expression class templates.
Header file for the Size type trait.
Header file for the reset shim.
Header file for the serial shim.
Base class for all compute expression templates.
Definition: Computation.h:68
Base template for the DeclDiagTrait class.
Definition: DeclDiagTrait.h:127
Generic wrapper for the decldiag() function.
Definition: DeclDiag.h:61
Base template for the DeclHermTrait class.
Definition: DeclHermTrait.h:126
Generic wrapper for the declherm() function.
Definition: DeclHerm.h:61
Base template for the DeclLowTrait class.
Definition: DeclLowTrait.h:126
Generic wrapper for the decllow() function.
Definition: DeclLow.h:61
Base template for the DeclSymTrait class.
Definition: DeclSymTrait.h:126
Generic wrapper for the declsym() function.
Definition: DeclSym.h:61
Base template for the DeclUppTrait class.
Definition: DeclUppTrait.h:126
Generic wrapper for the declupp() function.
Definition: DeclUpp.h:61
Base class for all matrix/matrix multiplication expression templates.
Definition: MatMatMultExpr.h:71
Base template for the MultTrait class.
Definition: MultTrait.h:130
Generic wrapper for the null function.
Definition: Noop.h:62
System settings for performance optimizations.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
Header file for the IsZero type trait.
Header file for the RequiresEvaluation type trait.
Header file for basic type definitions.
Header file for the generic max algorithm.
Header file for the generic min algorithm.