Blaze 3.9
TDMatSMatMultExpr.h
Go to the documentation of this file.
1//=================================================================================================
33//=================================================================================================
34
35#ifndef _BLAZE_MATH_EXPRESSIONS_TDMATSMATMULTEXPR_H_
36#define _BLAZE_MATH_EXPRESSIONS_TDMATSMATMULTEXPR_H_
37
38
39//*************************************************************************************************
40// Includes
41//*************************************************************************************************
42
43#include <blaze/math/Aliases.h>
96#include <blaze/util/Assert.h>
97#include <blaze/util/EnableIf.h>
101#include <blaze/util/mpl/If.h>
102#include <blaze/util/Types.h>
104
105
106namespace blaze {
107
108//=================================================================================================
109//
110// CLASS TDMATSMATMULTEXPR
111//
112//=================================================================================================
113
114//*************************************************************************************************
121template< typename MT1 // Type of the left-hand side dense matrix
122 , typename MT2 // Type of the right-hand side sparse matrix
123 , bool SF // Symmetry flag
124 , bool HF // Hermitian flag
125 , bool LF // Lower flag
126 , bool UF > // Upper flag
128 : public MatMatMultExpr< DenseMatrix< TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, true > >
129 , private Computation
130{
131 private:
132 //**Type definitions****************************************************************************
139 //**********************************************************************************************
140
141 //**********************************************************************************************
143 static constexpr bool evaluateLeft = ( IsComputation_v<MT1> || RequiresEvaluation_v<MT1> );
144 //**********************************************************************************************
145
146 //**********************************************************************************************
148 static constexpr bool evaluateRight = ( IsComputation_v<MT2> || RequiresEvaluation_v<MT2> );
149 //**********************************************************************************************
150
151 //**********************************************************************************************
152 static constexpr bool SYM = ( SF && !( HF || LF || UF ) );
153 static constexpr bool HERM = ( HF && !( LF || UF ) );
154 static constexpr bool LOW = ( LF || ( ( SF || HF ) && UF ) );
155 static constexpr bool UPP = ( UF || ( ( SF || HF ) && LF ) );
156 //**********************************************************************************************
157
158 //**********************************************************************************************
160
165 template< typename T1, typename T2, typename T3 >
166 static constexpr bool CanExploitSymmetry_v = IsSymmetric_v<T3>;
168 //**********************************************************************************************
169
170 //**********************************************************************************************
172
176 template< typename T1, typename T2, typename T3 >
177 static constexpr bool IsEvaluationRequired_v =
178 ( ( evaluateLeft || evaluateRight ) && !CanExploitSymmetry_v<T1,T2,T3> );
180 //**********************************************************************************************
181
182 //**********************************************************************************************
184
187 template< typename T1, typename T2, typename T3 >
188 static constexpr bool UseOptimizedKernel_v =
189 ( useOptimizedKernels &&
190 !IsDiagonal_v<T2> &&
191 !IsResizable_v< ElementType_t<T1> > &&
192 !( IsColumnMajorMatrix_v<T1> && IsResizable_v<ET2> ) );
194 //**********************************************************************************************
195
196 //**********************************************************************************************
198
201 template< typename T1, typename T2, typename T3 >
202 static constexpr bool UseDefaultKernel_v = !UseOptimizedKernel_v<T1,T2,T3>;
204 //**********************************************************************************************
205
206 //**********************************************************************************************
208
211 using ForwardFunctor = If_t< HERM
212 , DeclHerm
213 , If_t< SYM
214 , DeclSym
215 , If_t< LOW
216 , If_t< UPP
217 , DeclDiag
218 , DeclLow >
219 , If_t< UPP
220 , DeclUpp
221 , Noop > > > >;
223 //**********************************************************************************************
224
225 public:
226 //**Type definitions****************************************************************************
229
232
234 using ResultType = typename If_t< HERM
236 , If_t< SYM
238 , If_t< LOW
239 , If_t< UPP
242 , If_t< UPP
244 , MultTrait<RT1,RT2> > > > >::Type;
245
249 using ReturnType = const ElementType;
250 using CompositeType = const ResultType;
251
253 using LeftOperand = If_t< IsExpression_v<MT1>, const MT1, const MT1& >;
254
256 using RightOperand = If_t< IsExpression_v<MT2>, const MT2, const MT2& >;
257
260
263 //**********************************************************************************************
264
265 //**Compilation flags***************************************************************************
267 static constexpr bool simdEnabled = false;
268
270 static constexpr bool smpAssignable =
271 ( !evaluateLeft && MT1::smpAssignable && !evaluateRight && MT2::smpAssignable );
272 //**********************************************************************************************
273
274 //**Constructor*********************************************************************************
280 inline TDMatSMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
281 : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
282 , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
283 {
284 BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
285 }
286 //**********************************************************************************************
287
288 //**Access operator*****************************************************************************
295 inline ReturnType operator()( size_t i, size_t j ) const {
296 BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
297 BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
298
299 if( IsDiagonal_v<MT1> ) {
300 return lhs_(i,i) * rhs_(i,j);
301 }
302 else if( IsDiagonal_v<MT2> ) {
303 return lhs_(i,j) * rhs_(j,j);
304 }
305 else if( IsTriangular_v<MT1> || IsTriangular_v<MT2> ) {
306 const size_t begin( ( IsUpper_v<MT1> )
307 ?( ( IsLower_v<MT2> )
308 ?( max( ( IsStrictlyUpper_v<MT1> ? i+1UL : i )
309 , ( IsStrictlyLower_v<MT2> ? j+1UL : j ) ) )
310 :( IsStrictlyUpper_v<MT1> ? i+1UL : i ) )
311 :( ( IsLower_v<MT2> )
312 ?( IsStrictlyLower_v<MT2> ? j+1UL : j )
313 :( 0UL ) ) );
314 const size_t end( ( IsLower_v<MT1> )
315 ?( ( IsUpper_v<MT2> )
316 ?( min( ( IsStrictlyLower_v<MT1> ? i : i+1UL )
317 , ( IsStrictlyUpper_v<MT2> ? j : j+1UL ) ) )
318 :( IsStrictlyLower_v<MT1> ? i : i+1UL ) )
319 :( ( IsUpper_v<MT2> )
320 ?( IsStrictlyUpper_v<MT2> ? j : j+1UL )
321 :( lhs_.columns() ) ) );
322
323 if( begin >= end ) return ElementType();
324
325 const size_t n( end - begin );
326
327 return subvector( row( lhs_, i, unchecked ), begin, n, unchecked ) *
329 }
330 else {
331 return row( lhs_, i, unchecked ) * column( rhs_, j, unchecked );
332 }
333 }
334 //**********************************************************************************************
335
336 //**At function*********************************************************************************
344 inline ReturnType at( size_t i, size_t j ) const {
345 if( i >= lhs_.rows() ) {
346 BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
347 }
348 if( j >= rhs_.columns() ) {
349 BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
350 }
351 return (*this)(i,j);
352 }
353 //**********************************************************************************************
354
355 //**Rows function*******************************************************************************
360 inline size_t rows() const noexcept {
361 return lhs_.rows();
362 }
363 //**********************************************************************************************
364
365 //**Columns function****************************************************************************
370 inline size_t columns() const noexcept {
371 return rhs_.columns();
372 }
373 //**********************************************************************************************
374
375 //**Left operand access*************************************************************************
380 inline LeftOperand leftOperand() const noexcept {
381 return lhs_;
382 }
383 //**********************************************************************************************
384
385 //**Right operand access************************************************************************
390 inline RightOperand rightOperand() const noexcept {
391 return rhs_;
392 }
393 //**********************************************************************************************
394
395 //**********************************************************************************************
401 template< typename T >
402 inline bool canAlias( const T* alias ) const noexcept {
403 return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
404 }
405 //**********************************************************************************************
406
407 //**********************************************************************************************
413 template< typename T >
414 inline bool isAliased( const T* alias ) const {
415 return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
416 }
417 //**********************************************************************************************
418
419 //**********************************************************************************************
424 inline bool isAligned() const noexcept {
425 return lhs_.isAligned();
426 }
427 //**********************************************************************************************
428
429 //**********************************************************************************************
434 inline bool canSMPAssign() const noexcept {
435 return ( rows() * columns() >= SMP_TDMATSMATMULT_THRESHOLD ) && !IsDiagonal_v<MT1>;
436 }
437 //**********************************************************************************************
438
439 private:
440 //**Member variables****************************************************************************
443 //**********************************************************************************************
444
445 //**Assignment to dense matrices****************************************************************
458 template< typename MT // Type of the target dense matrix
459 , bool SO > // Storage order of the target dense matrix
460 friend inline auto assign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
462 {
464
465 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
466 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
467
468 LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
469 RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
470
471 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
472 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
473 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
474 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
475 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
476 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
477
478 TDMatSMatMultExpr::selectAssignKernel( *lhs, A, B );
479 }
481 //**********************************************************************************************
482
483 //**Assignment to dense matrices (kernel selection)*********************************************
494 template< typename MT3 // Type of the left-hand side target matrix
495 , typename MT4 // Type of the left-hand side matrix operand
496 , typename MT5 > // Type of the right-hand side matrix operand
497 static inline void selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
498 {
499 if( C.rows() * C.columns() < TDMATSMATMULT_THRESHOLD )
500 selectSmallAssignKernel( C, A, B );
501 else
502 selectLargeAssignKernel( C, A, B );
503 }
505 //**********************************************************************************************
506
507 //**Default assignment to dense matrices********************************************************
522 template< typename MT3 // Type of the left-hand side target matrix
523 , typename MT4 // Type of the left-hand side matrix operand
524 , typename MT5 > // Type of the right-hand side matrix operand
525 static inline void selectDefaultAssignKernel( MT3& C, const MT4& A, const MT5& B )
526 {
527 reset( C );
528
529 for( size_t j=0UL; j<B.rows(); ++j )
530 {
531 auto element( B.begin(j) );
532 const auto end( B.end(j) );
533
534 if( IsDiagonal_v<MT4> )
535 {
536 for( ; element!=end; ++element ) {
537 C(j,element->index()) = A(j,j) * element->value();
538 }
539 }
540 else
541 {
542 for( ; element!=end; ++element )
543 {
544 const size_t j1( element->index() );
545
546 const size_t ibegin( ( IsLower_v<MT4> )
547 ?( ( IsStrictlyLower_v<MT4> )
548 ?( SYM || HERM || LOW ? max(j1,j+1UL) : j+1UL )
549 :( SYM || HERM || LOW ? max(j1,j) : j ) )
550 :( SYM || HERM || LOW ? j1 : 0UL ) );
551 const size_t iend( ( IsUpper_v<MT4> )
552 ?( ( IsStrictlyUpper_v<MT4> )
553 ?( UPP ? min(j1+1UL,j) : j )
554 :( UPP ? min(j1,j)+1UL : j+1UL ) )
555 :( UPP ? j1+1UL : A.rows() ) );
556
557 if( ( SYM || HERM || LOW || UPP ) && ( ibegin >= iend ) ) continue;
558 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
559
560 for( size_t i=ibegin; i<iend; ++i ) {
561 if( isDefault( C(i,j1) ) )
562 C(i,j1) = A(i,j) * element->value();
563 else
564 C(i,j1) += A(i,j) * element->value();
565 }
566 }
567 }
568 }
569
570 if( SYM || HERM ) {
571 for( size_t j=1UL; j<B.columns(); ++j ) {
572 for( size_t i=0UL; i<j; ++i ) {
573 C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
574 }
575 }
576 }
577 }
579 //**********************************************************************************************
580
581 //**Default assignment to dense matrices (small matrices)***************************************
595 template< typename MT3 // Type of the left-hand side target matrix
596 , typename MT4 // Type of the left-hand side matrix operand
597 , typename MT5 > // Type of the right-hand side matrix operand
598 static inline auto selectSmallAssignKernel( MT3& C, const MT4& A, const MT5& B )
599 -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
600 {
601 selectDefaultAssignKernel( C, A, B );
602 }
604 //**********************************************************************************************
605
606 //**Optimized assignment to dense matrices (small matrices)*************************************
621 template< typename MT3 // Type of the left-hand side target matrix
622 , typename MT4 // Type of the left-hand side matrix operand
623 , typename MT5 > // Type of the right-hand side matrix operand
624 static inline auto selectSmallAssignKernel( MT3& C, const MT4& A, const MT5& B )
625 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
626 {
627 reset( C );
628
629 for( size_t j=0UL; j<B.rows(); ++j )
630 {
631 auto element( B.begin(j) );
632 const auto end( B.end(j) );
633
634 const size_t nonzeros( B.nonZeros(j) );
635 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
636 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
637
638 for( size_t k=0UL; k<kpos; k+=4UL )
639 {
640 const size_t j1( element->index() );
641 const ET2 v1( element->value() );
642 ++element;
643 const size_t j2( element->index() );
644 const ET2 v2( element->value() );
645 ++element;
646 const size_t j3( element->index() );
647 const ET2 v3( element->value() );
648 ++element;
649 const size_t j4( element->index() );
650 const ET2 v4( element->value() );
651 ++element;
652
653 BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
654
655 const size_t ibegin( ( IsLower_v<MT4> )
656 ?( ( IsStrictlyLower_v<MT4> )
657 ?( SYM || HERM || LOW ? max(j1,j+1UL) : j+1UL )
658 :( SYM || HERM || LOW ? max(j1,j) : j ) )
659 :( SYM || HERM || LOW ? j1 : 0UL ) );
660 const size_t iend( ( IsUpper_v<MT4> )
661 ?( ( IsStrictlyUpper_v<MT4> )
662 ?( UPP ? min(j4+1UL,j) : j )
663 :( UPP ? min(j4,j)+1UL : j+1UL ) )
664 :( UPP ? j4+1UL : A.rows() ) );
665
666 if( ( SYM || HERM || LOW || UPP ) && ( ibegin >= iend ) ) continue;
667 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
668
669 const size_t inum( iend - ibegin );
670 const size_t ipos( ibegin + prevMultiple( inum, 4UL ) );
671 BLAZE_INTERNAL_ASSERT( ipos <= ibegin+inum, "Invalid end calculation" );
672
673 size_t i( ibegin );
674
675 for( ; i<ipos; i+=4UL ) {
676 C(i ,j1) += A(i ,j) * v1;
677 C(i+1UL,j1) += A(i+1UL,j) * v1;
678 C(i+2UL,j1) += A(i+2UL,j) * v1;
679 C(i+3UL,j1) += A(i+3UL,j) * v1;
680 C(i ,j2) += A(i ,j) * v2;
681 C(i+1UL,j2) += A(i+1UL,j) * v2;
682 C(i+2UL,j2) += A(i+2UL,j) * v2;
683 C(i+3UL,j2) += A(i+3UL,j) * v2;
684 C(i ,j3) += A(i ,j) * v3;
685 C(i+1UL,j3) += A(i+1UL,j) * v3;
686 C(i+2UL,j3) += A(i+2UL,j) * v3;
687 C(i+3UL,j3) += A(i+3UL,j) * v3;
688 C(i ,j4) += A(i ,j) * v4;
689 C(i+1UL,j4) += A(i+1UL,j) * v4;
690 C(i+2UL,j4) += A(i+2UL,j) * v4;
691 C(i+3UL,j4) += A(i+3UL,j) * v4;
692 }
693 for( ; i<iend; ++i ) {
694 C(i,j1) += A(i,j) * v1;
695 C(i,j2) += A(i,j) * v2;
696 C(i,j3) += A(i,j) * v3;
697 C(i,j4) += A(i,j) * v4;
698 }
699 }
700
701 for( ; element!=end; ++element )
702 {
703 const size_t j1( element->index() );
704 const ET2 v1( element->value() );
705
706 const size_t ibegin( ( IsLower_v<MT4> )
707 ?( ( IsStrictlyLower_v<MT4> )
708 ?( SYM || HERM || LOW ? max(j1,j+1UL) : j+1UL )
709 :( SYM || HERM || LOW ? max(j1,j) : j ) )
710 :( SYM || HERM || LOW ? j1 : 0UL ) );
711 const size_t iend( ( IsUpper_v<MT4> )
712 ?( ( IsStrictlyUpper_v<MT4> )
713 ?( UPP ? min(j1+1UL,j) : j )
714 :( UPP ? min(j1,j)+1UL : j+1UL ) )
715 :( UPP ? j1+1UL : A.rows() ) );
716
717 if( ( SYM || HERM || LOW || UPP ) && ( ibegin >= iend ) ) continue;
718 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
719
720 const size_t inum( iend - ibegin );
721 const size_t ipos( ibegin + prevMultiple( inum, 4UL ) );
722 BLAZE_INTERNAL_ASSERT( ipos <= ibegin+inum, "Invalid end calculation" );
723
724 size_t i( ibegin );
725
726 for( ; i<ipos; i+=4UL ) {
727 C(i ,j1) += A(i ,j) * v1;
728 C(i+1UL,j1) += A(i+1UL,j) * v1;
729 C(i+2UL,j1) += A(i+2UL,j) * v1;
730 C(i+3UL,j1) += A(i+3UL,j) * v1;
731 }
732 for( ; i<iend; ++i ) {
733 C(i,j1) += A(i,j) * v1;
734 }
735 }
736 }
737
738 if( SYM || HERM ) {
739 for( size_t j=1UL; j<B.columns(); ++j ) {
740 for( size_t i=0UL; i<j; ++i ) {
741 C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
742 }
743 }
744 }
745 }
747 //**********************************************************************************************
748
749 //**Default assignment to dense matrices (large matrices)***************************************
763 template< typename MT3 // Type of the left-hand side target matrix
764 , typename MT4 // Type of the left-hand side matrix operand
765 , typename MT5 > // Type of the right-hand side matrix operand
766 static inline auto selectLargeAssignKernel( MT3& C, const MT4& A, const MT5& B )
767 -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
768 {
769 selectDefaultAssignKernel( C, A, B );
770 }
772 //**********************************************************************************************
773
774 //**Optimized assignment to dense matrices (large matrices)*************************************
789 template< typename MT3 // Type of the left-hand side target matrix
790 , typename MT4 // Type of the left-hand side matrix operand
791 , typename MT5 > // Type of the right-hand side matrix operand
792 static inline auto selectLargeAssignKernel( MT3& C, const MT4& A, const MT5& B )
793 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
794 {
796
797 const ForwardFunctor fwd;
798
799 const OppositeType_t<MT5> tmp( serial( B ) );
800 assign( C, fwd( A * tmp ) );
801 }
803 //**********************************************************************************************
804
805 //**Assignment to sparse matrices***************************************************************
818 template< typename MT // Type of the target sparse matrix
819 , bool SO > // Storage order of the target sparse matrix
820 friend inline auto assign( SparseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
821 -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
822 {
824
825 using TmpType = If_t< SO, ResultType, OppositeType >;
826
833
834 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
835 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
836
837 const ForwardFunctor fwd;
838
839 const TmpType tmp( serial( rhs ) );
840 assign( *lhs, fwd( tmp ) );
841 }
843 //**********************************************************************************************
844
845 //**Restructuring assignment********************************************************************
860 template< typename MT // Type of the target matrix
861 , bool SO > // Storage order of the target matrix
862 friend inline auto assign( Matrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
863 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
864 {
866
868
869 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
870 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
871
872 const ForwardFunctor fwd;
873
874 assign( *lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
875 }
877 //**********************************************************************************************
878
879 //**Addition assignment to dense matrices*******************************************************
892 template< typename MT // Type of the target dense matrix
893 , bool SO > // Storage order of the target dense matrix
894 friend inline auto addAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
895 -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
896 {
898
899 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
900 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
901
902 LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
903 RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
904
905 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
906 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
907 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
908 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
909 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
910 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
911
912 TDMatSMatMultExpr::selectAddAssignKernel( *lhs, A, B );
913 }
915 //**********************************************************************************************
916
917 //**Addition assignment to dense matrices (kernel selection)************************************
928 template< typename MT3 // Type of the left-hand side target matrix
929 , typename MT4 // Type of the left-hand side matrix operand
930 , typename MT5 > // Type of the right-hand side matrix operand
931 static inline void selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
932 {
933 if( C.rows() * C.columns() < TDMATSMATMULT_THRESHOLD )
934 selectSmallAddAssignKernel( C, A, B );
935 else
936 selectLargeAddAssignKernel( C, A, B );
937 }
939 //**********************************************************************************************
940
941 //**Default addition assignment to dense matrices***********************************************
956 template< typename MT3 // Type of the left-hand side target matrix
957 , typename MT4 // Type of the left-hand side matrix operand
958 , typename MT5 > // Type of the right-hand side matrix operand
959 static inline void selectDefaultAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
960 {
961 size_t i( 0UL );
962
963 for( size_t j=0UL; j<B.rows(); ++j )
964 {
965 auto element( B.begin(j) );
966 const auto end( B.end(j) );
967
968 if( IsDiagonal_v<MT4> )
969 {
970 for( ; element!=end; ++element ) {
971 C(j,element->index()) += A(j,j) * element->value();
972 }
973 }
974 else
975 {
976 for( ; element!=end; ++element )
977 {
978 const size_t j1( element->index() );
979
980 const size_t ibegin( ( IsLower_v<MT4> )
981 ?( ( IsStrictlyLower_v<MT4> )
982 ?( LOW ? max(j1,j+1UL) : j+1UL )
983 :( LOW ? max(j1,j) : j ) )
984 :( LOW ? j1 : 0UL ) );
985 const size_t iend( ( IsUpper_v<MT4> )
986 ?( ( IsStrictlyUpper_v<MT4> )
987 ?( UPP ? min(j1+1UL,j) : j )
988 :( UPP ? min(j1,j)+1UL : j+1UL ) )
989 :( UPP ? j1+1UL : A.rows() ) );
990
991 if( ( LOW || UPP ) && ( ibegin >= iend ) ) continue;
992 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
993
994 const size_t inum( iend - ibegin );
995 const size_t ipos( ibegin + prevMultiple( inum, 4UL ) );
996 BLAZE_INTERNAL_ASSERT( ipos <= ibegin+inum, "Invalid end calculation" );
997
998 for( i=ibegin; i<ipos; i+=4UL ) {
999 C(i ,j1) += A(i ,j) * element->value();
1000 C(i+1UL,j1) += A(i+1UL,j) * element->value();
1001 C(i+2UL,j1) += A(i+2UL,j) * element->value();
1002 C(i+3UL,j1) += A(i+3UL,j) * element->value();
1003 }
1004 for( ; i<iend; ++i ) {
1005 C(i,j1) += A(i,j) * element->value();
1006 }
1007 }
1008 }
1009 }
1010 }
1012 //**********************************************************************************************
1013
1014 //**Default addition assignment to dense matrices (small matrices)******************************
1028 template< typename MT3 // Type of the left-hand side target matrix
1029 , typename MT4 // Type of the left-hand side matrix operand
1030 , typename MT5 > // Type of the right-hand side matrix operand
1031 static inline auto selectSmallAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1032 -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
1033 {
1034 selectDefaultAddAssignKernel( C, A, B );
1035 }
1037 //**********************************************************************************************
1038
1039 //**Optimized addition assignment to dense matrices (small matrices)****************************
1054 template< typename MT3 // Type of the left-hand side target matrix
1055 , typename MT4 // Type of the left-hand side matrix operand
1056 , typename MT5 > // Type of the right-hand side matrix operand
1057 static inline auto selectSmallAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1058 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1059 {
1060 for( size_t j=0UL; j<B.rows(); ++j )
1061 {
1062 auto element( B.begin(j) );
1063 const auto end( B.end(j) );
1064
1065 const size_t nonzeros( B.nonZeros(j) );
1066 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1067 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1068
1069 for( size_t k=0UL; k<kpos; k+=4UL )
1070 {
1071 const size_t j1( element->index() );
1072 const ET2 v1( element->value() );
1073 ++element;
1074 const size_t j2( element->index() );
1075 const ET2 v2( element->value() );
1076 ++element;
1077 const size_t j3( element->index() );
1078 const ET2 v3( element->value() );
1079 ++element;
1080 const size_t j4( element->index() );
1081 const ET2 v4( element->value() );
1082 ++element;
1083
1084 BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1085
1086 const size_t ibegin( ( IsLower_v<MT4> )
1087 ?( ( IsStrictlyLower_v<MT4> )
1088 ?( LOW ? max(j1,j+1UL) : j+1UL )
1089 :( LOW ? max(j1,j) : j ) )
1090 :( LOW ? j1 : 0UL ) );
1091 const size_t iend( ( IsUpper_v<MT4> )
1092 ?( ( IsStrictlyUpper_v<MT4> )
1093 ?( UPP ? min(j4+1UL,j) : j )
1094 :( UPP ? min(j4,j)+1UL : j+1UL ) )
1095 :( UPP ? j4+1UL : A.rows() ) );
1096
1097 if( ( LOW || UPP ) && ( ibegin >= iend ) ) continue;
1098 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1099
1100 const size_t inum( iend - ibegin );
1101 const size_t ipos( ibegin + prevMultiple( inum, 4UL ) );
1102 BLAZE_INTERNAL_ASSERT( ipos <= ibegin+inum, "Invalid end calculation" );
1103
1104 size_t i( ibegin );
1105
1106 for( i=ibegin; i<ipos; i+=4UL ) {
1107 C(i ,j1) += A(i ,j) * v1;
1108 C(i+1UL,j1) += A(i+1UL,j) * v1;
1109 C(i+2UL,j1) += A(i+2UL,j) * v1;
1110 C(i+3UL,j1) += A(i+3UL,j) * v1;
1111 C(i ,j2) += A(i ,j) * v2;
1112 C(i+1UL,j2) += A(i+1UL,j) * v2;
1113 C(i+2UL,j2) += A(i+2UL,j) * v2;
1114 C(i+3UL,j2) += A(i+3UL,j) * v2;
1115 C(i ,j3) += A(i ,j) * v3;
1116 C(i+1UL,j3) += A(i+1UL,j) * v3;
1117 C(i+2UL,j3) += A(i+2UL,j) * v3;
1118 C(i+3UL,j3) += A(i+3UL,j) * v3;
1119 C(i ,j4) += A(i ,j) * v4;
1120 C(i+1UL,j4) += A(i+1UL,j) * v4;
1121 C(i+2UL,j4) += A(i+2UL,j) * v4;
1122 C(i+3UL,j4) += A(i+3UL,j) * v4;
1123 }
1124 for( ; i<iend; ++i ) {
1125 C(i,j1) += A(i,j) * v1;
1126 C(i,j2) += A(i,j) * v2;
1127 C(i,j3) += A(i,j) * v3;
1128 C(i,j4) += A(i,j) * v4;
1129 }
1130 }
1131
1132 for( ; element!=end; ++element )
1133 {
1134 const size_t j1( element->index() );
1135 const ET2 v1( element->value() );
1136
1137 const size_t ibegin( ( IsLower_v<MT4> )
1138 ?( ( IsStrictlyLower_v<MT4> )
1139 ?( LOW ? max(j1,j+1UL) : j+1UL )
1140 :( LOW ? max(j1,j) : j ) )
1141 :( LOW ? j1 : 0UL ) );
1142 const size_t iend( ( IsUpper_v<MT4> )
1143 ?( ( IsStrictlyUpper_v<MT4> )
1144 ?( UPP ? min(j1+1UL,j) : j )
1145 :( UPP ? min(j1,j)+1UL : j+1UL ) )
1146 :( UPP ? j1+1UL : A.rows() ) );
1147
1148 if( ( LOW || UPP ) && ( ibegin >= iend ) ) continue;
1149 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1150
1151 const size_t inum( iend - ibegin );
1152 const size_t ipos( ibegin + prevMultiple( inum, 4UL ) );
1153 BLAZE_INTERNAL_ASSERT( ipos <= ibegin+inum, "Invalid end calculation" );
1154
1155 size_t i( ibegin );
1156
1157 for( ; i<ipos; i+=4UL ) {
1158 C(i ,j1) += A(i ,j) * v1;
1159 C(i+1UL,j1) += A(i+1UL,j) * v1;
1160 C(i+2UL,j1) += A(i+2UL,j) * v1;
1161 C(i+3UL,j1) += A(i+3UL,j) * v1;
1162 }
1163 for( ; i<iend; ++i ) {
1164 C(i,j1) += A(i,j) * v1;
1165 }
1166 }
1167 }
1168 }
1170 //**********************************************************************************************
1171
1172 //**Default addition assignment to dense matrices (large matrices)******************************
1186 template< typename MT3 // Type of the left-hand side target matrix
1187 , typename MT4 // Type of the left-hand side matrix operand
1188 , typename MT5 > // Type of the right-hand side matrix operand
1189 static inline auto selectLargeAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1190 -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
1191 {
1192 selectDefaultAddAssignKernel( C, A, B );
1193 }
1195 //**********************************************************************************************
1196
1197 //**Optimized addition assignment to dense matrices (large matrices)****************************
1212 template< typename MT3 // Type of the left-hand side target matrix
1213 , typename MT4 // Type of the left-hand side matrix operand
1214 , typename MT5 > // Type of the right-hand side matrix operand
1215 static inline auto selectLargeAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1216 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1217 {
1219
1220 const ForwardFunctor fwd;
1221
1222 const OppositeType_t<MT5> tmp( serial( B ) );
1223 addAssign( C, fwd( A * tmp ) );
1224 }
1226 //**********************************************************************************************
1227
1228 //**Restructuring addition assignment***********************************************************
1243 template< typename MT // Type of the target matrix
1244 , bool SO > // Storage order of the target matrix
1245 friend inline auto addAssign( Matrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1246 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1247 {
1249
1250 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1251 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1252
1253 const ForwardFunctor fwd;
1254
1255 addAssign( *lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1256 }
1258 //**********************************************************************************************
1259
1260 //**Addition assignment to sparse matrices******************************************************
1261 // No special implementation for the addition assignment to sparse matrices.
1262 //**********************************************************************************************
1263
1264 //**Subtraction assignment to dense matrices****************************************************
1277 template< typename MT // Type of the target dense matrix
1278 , bool SO > // Storage order of the target dense matrix
1279 friend inline auto subAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1280 -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1281 {
1283
1284 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1285 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1286
1287 LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1288 RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1289
1290 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1291 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1292 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1293 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1294 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1295 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
1296
1297 TDMatSMatMultExpr::selectSubAssignKernel( *lhs, A, B );
1298 }
1300 //**********************************************************************************************
1301
1302 //**Subtraction assignment to dense matrices (kernel selection)*********************************
1313 template< typename MT3 // Type of the left-hand side target matrix
1314 , typename MT4 // Type of the left-hand side matrix operand
1315 , typename MT5 > // Type of the right-hand side matrix operand
1316 static inline void selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1317 {
1318 if( C.rows() * C.columns() < TDMATSMATMULT_THRESHOLD )
1319 selectSmallSubAssignKernel( C, A, B );
1320 else
1321 selectLargeSubAssignKernel( C, A, B );
1322 }
1324 //**********************************************************************************************
1325
1326 //**Default subtraction assignment to dense matrices********************************************
1341 template< typename MT3 // Type of the left-hand side target matrix
1342 , typename MT4 // Type of the left-hand side matrix operand
1343 , typename MT5 > // Type of the right-hand side matrix operand
1344 static inline void selectDefaultSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1345 {
1346 size_t i( 0UL );
1347
1348 for( size_t j=0UL; j<B.rows(); ++j )
1349 {
1350 auto element( B.begin(j) );
1351 const auto end( B.end(j) );
1352
1353 if( IsDiagonal_v<MT4> )
1354 {
1355 for( ; element!=end; ++element ) {
1356 C(j,element->index()) -= A(j,j) * element->value();
1357 }
1358 }
1359 else
1360 {
1361 for( ; element!=end; ++element )
1362 {
1363 const size_t j1( element->index() );
1364
1365 const size_t ibegin( ( IsLower_v<MT4> )
1366 ?( ( IsStrictlyLower_v<MT4> )
1367 ?( LOW ? max(j1,j+1UL) : j+1UL )
1368 :( LOW ? max(j1,j) : j ) )
1369 :( LOW ? j1 : 0UL ) );
1370 const size_t iend( ( IsUpper_v<MT4> )
1371 ?( ( IsStrictlyUpper_v<MT4> )
1372 ?( UPP ? min(j1+1UL,j) : j )
1373 :( UPP ? min(j1,j)+1UL : j+1UL ) )
1374 :( UPP ? j1+1UL : A.rows() ) );
1375
1376 if( ( LOW || UPP ) && ( ibegin >= iend ) ) continue;
1377 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1378
1379 const size_t inum( iend - ibegin );
1380 const size_t ipos( ibegin + prevMultiple( inum, 4UL ) );
1381 BLAZE_INTERNAL_ASSERT( ipos <= ibegin+inum, "Invalid end calculation" );
1382
1383 for( i=ibegin; i<ipos; i+=4UL ) {
1384 C(i ,j1) -= A(i ,j) * element->value();
1385 C(i+1UL,j1) -= A(i+1UL,j) * element->value();
1386 C(i+2UL,j1) -= A(i+2UL,j) * element->value();
1387 C(i+3UL,j1) -= A(i+3UL,j) * element->value();
1388 }
1389 for( ; i<iend; ++i ) {
1390 C(i,j1) -= A(i,j) * element->value();
1391 }
1392 }
1393 }
1394 }
1395 }
1397 //**********************************************************************************************
1398
1399 //**Default subtraction assignment to dense matrices (small matrices)***************************
1413 template< typename MT3 // Type of the left-hand side target matrix
1414 , typename MT4 // Type of the left-hand side matrix operand
1415 , typename MT5 > // Type of the right-hand side matrix operand
1416 static inline auto selectSmallSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1417 -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
1418 {
1419 selectDefaultSubAssignKernel( C, A, B );
1420 }
1422 //**********************************************************************************************
1423
1424 //**Optimized subtraction assignment to dense matrices (small matrices)*************************
1439 template< typename MT3 // Type of the left-hand side target matrix
1440 , typename MT4 // Type of the left-hand side matrix operand
1441 , typename MT5 > // Type of the right-hand side matrix operand
1442 static inline auto selectSmallSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1443 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1444 {
1445 for( size_t j=0UL; j<B.rows(); ++j )
1446 {
1447 auto element( B.begin(j) );
1448 const auto end( B.end(j) );
1449
1450 const size_t nonzeros( B.nonZeros(j) );
1451 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1452 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1453
1454 for( size_t k=0UL; k<kpos; k+=4UL )
1455 {
1456 const size_t j1( element->index() );
1457 const ET2 v1( element->value() );
1458 ++element;
1459 const size_t j2( element->index() );
1460 const ET2 v2( element->value() );
1461 ++element;
1462 const size_t j3( element->index() );
1463 const ET2 v3( element->value() );
1464 ++element;
1465 const size_t j4( element->index() );
1466 const ET2 v4( element->value() );
1467 ++element;
1468
1469 BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1470
1471 const size_t ibegin( ( IsLower_v<MT4> )
1472 ?( ( IsStrictlyLower_v<MT4> )
1473 ?( LOW ? max(j1,j+1UL) : j+1UL )
1474 :( LOW ? max(j1,j) : j ) )
1475 :( LOW ? j1 : 0UL ) );
1476 const size_t iend( ( IsUpper_v<MT4> )
1477 ?( ( IsStrictlyUpper_v<MT4> )
1478 ?( UPP ? min(j4+1UL,j) : j )
1479 :( UPP ? min(j4,j)+1UL : j+1UL ) )
1480 :( UPP ? j4+1UL : A.rows() ) );
1481
1482 if( ( LOW || UPP ) && ( ibegin >= iend ) ) continue;
1483 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1484
1485 const size_t inum( iend - ibegin );
1486 const size_t ipos( ibegin + prevMultiple( inum, 4UL ) );
1487 BLAZE_INTERNAL_ASSERT( ipos <= ibegin+inum, "Invalid end calculation" );
1488
1489 size_t i( ibegin );
1490
1491 for( ; i<ipos; i+=4UL ) {
1492 C(i ,j1) -= A(i ,j) * v1;
1493 C(i+1UL,j1) -= A(i+1UL,j) * v1;
1494 C(i+2UL,j1) -= A(i+2UL,j) * v1;
1495 C(i+3UL,j1) -= A(i+3UL,j) * v1;
1496 C(i ,j2) -= A(i ,j) * v2;
1497 C(i+1UL,j2) -= A(i+1UL,j) * v2;
1498 C(i+2UL,j2) -= A(i+2UL,j) * v2;
1499 C(i+3UL,j2) -= A(i+3UL,j) * v2;
1500 C(i ,j3) -= A(i ,j) * v3;
1501 C(i+1UL,j3) -= A(i+1UL,j) * v3;
1502 C(i+2UL,j3) -= A(i+2UL,j) * v3;
1503 C(i+3UL,j3) -= A(i+3UL,j) * v3;
1504 C(i ,j4) -= A(i ,j) * v4;
1505 C(i+1UL,j4) -= A(i+1UL,j) * v4;
1506 C(i+2UL,j4) -= A(i+2UL,j) * v4;
1507 C(i+3UL,j4) -= A(i+3UL,j) * v4;
1508 }
1509 for( ; i<iend; ++i ) {
1510 C(i,j1) -= A(i,j) * v1;
1511 C(i,j2) -= A(i,j) * v2;
1512 C(i,j3) -= A(i,j) * v3;
1513 C(i,j4) -= A(i,j) * v4;
1514 }
1515 }
1516
1517 for( ; element!=end; ++element )
1518 {
1519 const size_t j1( element->index() );
1520 const ET2 v1( element->value() );
1521
1522 const size_t ibegin( ( IsLower_v<MT4> )
1523 ?( ( IsStrictlyLower_v<MT4> )
1524 ?( LOW ? max(j1,j+1UL) : j+1UL )
1525 :( LOW ? max(j1,j) : j ) )
1526 :( LOW ? j1 : 0UL ) );
1527 const size_t iend( ( IsUpper_v<MT4> )
1528 ?( ( IsStrictlyUpper_v<MT4> )
1529 ?( UPP ? min(j1+1UL,j) : j )
1530 :( UPP ? min(j1,j)+1UL : j+1UL ) )
1531 :( UPP ? j1+1UL : A.rows() ) );
1532
1533 if( ( LOW || UPP ) && ( ibegin >= iend ) ) continue;
1534 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
1535
1536 const size_t inum( iend - ibegin );
1537 const size_t ipos( ibegin + prevMultiple( inum, 4UL ) );
1538 BLAZE_INTERNAL_ASSERT( ipos <= ibegin+inum, "Invalid end calculation" );
1539
1540 size_t i( ibegin );
1541
1542 for( ; i<ipos; i+=4UL ) {
1543 C(i ,j1) -= A(i ,j) * v1;
1544 C(i+1UL,j1) -= A(i+1UL,j) * v1;
1545 C(i+2UL,j1) -= A(i+2UL,j) * v1;
1546 C(i+3UL,j1) -= A(i+3UL,j) * v1;
1547 }
1548 for( ; i<iend; ++i ) {
1549 C(i,j1) -= A(i,j) * v1;
1550 }
1551 }
1552 }
1553 }
1555 //**********************************************************************************************
1556
1557 //**Default subtraction assignment to dense matrices (large matrices)***************************
1571 template< typename MT3 // Type of the left-hand side target matrix
1572 , typename MT4 // Type of the left-hand side matrix operand
1573 , typename MT5 > // Type of the right-hand side matrix operand
1574 static inline auto selectLargeSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1575 -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
1576 {
1577 selectDefaultSubAssignKernel( C, A, B );
1578 }
1580 //**********************************************************************************************
1581
1582 //**Optimized subtraction assignment to dense matrices (large matrices)*************************
1597 template< typename MT3 // Type of the left-hand side target matrix
1598 , typename MT4 // Type of the left-hand side matrix operand
1599 , typename MT5 > // Type of the right-hand side matrix operand
1600 static inline auto selectLargeSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1601 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1602 {
1604
1605 const ForwardFunctor fwd;
1606
1607 const OppositeType_t<MT5> tmp( serial( B ) );
1608 subAssign( C, fwd( A * tmp ) );
1609 }
1611 //**********************************************************************************************
1612
1613 //**Restructuring subtraction assignment********************************************************
1628 template< typename MT // Type of the target matrix
1629 , bool SO > // Storage order of the target matrix
1630 friend inline auto subAssign( Matrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1631 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1632 {
1634
1636
1637 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1638 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1639
1640 const ForwardFunctor fwd;
1641
1642 subAssign( *lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1643 }
1645 //**********************************************************************************************
1646
1647 //**Subtraction assignment to sparse matrices***************************************************
1648 // No special implementation for the subtraction assignment to sparse matrices.
1649 //**********************************************************************************************
1650
1651 //**Schur product assignment to dense matrices**************************************************
1664 template< typename MT // Type of the target dense matrix
1665 , bool SO > // Storage order of the target dense matrix
1666 friend inline void schurAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1667 {
1669
1673
1674 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1675 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1676
1677 const ResultType tmp( serial( rhs ) );
1678 schurAssign( *lhs, tmp );
1679 }
1681 //**********************************************************************************************
1682
1683 //**Schur product assignment to sparse matrices*************************************************
1684 // No special implementation for the Schur product assignment to sparse matrices.
1685 //**********************************************************************************************
1686
1687 //**Multiplication assignment to dense matrices*************************************************
1688 // No special implementation for the multiplication assignment to dense matrices.
1689 //**********************************************************************************************
1690
1691 //**Multiplication assignment to sparse matrices************************************************
1692 // No special implementation for the multiplication assignment to sparse matrices.
1693 //**********************************************************************************************
1694
1695 //**SMP assignment to dense matrices************************************************************
1711 template< typename MT // Type of the target dense matrix
1712 , bool SO > // Storage order of the target dense matrix
1713 friend inline auto smpAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1714 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1715 {
1717
1718 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1719 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1720
1721 LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1722 RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1723
1724 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1725 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1726 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1727 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1728 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1729 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
1730
1731 smpAssign( *lhs, A * B );
1732 }
1734 //**********************************************************************************************
1735
1736 //**SMP assignment to sparse matrices***********************************************************
1752 template< typename MT // Type of the target sparse matrix
1753 , bool SO > // Storage order of the target sparse matrix
1754 friend inline auto smpAssign( SparseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1755 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1756 {
1758
1759 using TmpType = If_t< SO, ResultType, OppositeType >;
1760
1767
1768 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1769 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1770
1771 const ForwardFunctor fwd;
1772
1773 const TmpType tmp( rhs );
1774 smpAssign( *lhs, fwd( tmp ) );
1775 }
1777 //**********************************************************************************************
1778
1779 //**Restructuring SMP assignment****************************************************************
1794 template< typename MT // Type of the target matrix
1795 , bool SO > // Storage order of the target matrix
1796 friend inline auto smpAssign( Matrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1797 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1798 {
1800
1801 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1802 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1803
1804 const ForwardFunctor fwd;
1805
1806 smpAssign( *lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1807 }
1809 //**********************************************************************************************
1810
1811 //**SMP addition assignment to dense matrices***************************************************
1827 template< typename MT // Type of the target dense matrix
1828 , bool SO > // Storage order of the target sparse matrix
1829 friend inline auto smpAddAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1830 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1831 {
1833
1834 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1835 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1836
1837 LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1838 RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1839
1840 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1841 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1842 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1843 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1844 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1845 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
1846
1847 smpAddAssign( *lhs, A * B );
1848 }
1850 //**********************************************************************************************
1851
1852 //**Restructuring SMP addition assignment*******************************************************
1867 template< typename MT // Type of the target matrix
1868 , bool SO > // Storage order of the target matrix
1869 friend inline auto smpAddAssign( Matrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1870 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1871 {
1873
1875
1876 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1877 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1878
1879 const ForwardFunctor fwd;
1880
1881 smpAddAssign( *lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1882 }
1884 //**********************************************************************************************
1885
1886 //**SMP addition assignment to sparse matrices**************************************************
1887 // No special implementation for the SMP addition assignment to sparse matrices.
1888 //**********************************************************************************************
1889
1890 //**SMP subtraction assignment to dense matrices************************************************
1906 template< typename MT // Type of the target dense matrix
1907 , bool SO > // Storage order of the target sparse matrix
1908 friend inline auto smpSubAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1909 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1910 {
1912
1913 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1914 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1915
1916 LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1917 RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1918
1919 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1920 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1921 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1922 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1923 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1924 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
1925
1926 smpSubAssign( *lhs, A * B );
1927 }
1929 //**********************************************************************************************
1930
1931 //**Restructuring SMP subtraction assignment****************************************************
1946 template< typename MT // Type of the target matrix
1947 , bool SO > // Storage order of the target matrix
1948 friend inline auto smpSubAssign( Matrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1949 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1950 {
1952
1954
1955 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1956 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1957
1958 const ForwardFunctor fwd;
1959
1960 smpSubAssign( *lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1961 }
1963 //**********************************************************************************************
1964
1965 //**SMP subtraction assignment to sparse matrices***********************************************
1966 // No special implementation for the SMP subtraction assignment to sparse matrices.
1967 //**********************************************************************************************
1968
1969 //**SMP Schur product assignment to dense matrices**********************************************
1982 template< typename MT // Type of the target dense matrix
1983 , bool SO > // Storage order of the target sparse matrix
1984 friend inline void smpSchurAssign( DenseMatrix<MT,SO>& lhs, const TDMatSMatMultExpr& rhs )
1985 {
1987
1991
1992 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1993 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1994
1995 const ResultType tmp( rhs );
1996 smpSchurAssign( *lhs, tmp );
1997 }
1999 //**********************************************************************************************
2000
2001 //**SMP Schur product assignment to sparse matrices*********************************************
2002 // No special implementation for the SMP Schur product assignment to sparse matrices.
2003 //**********************************************************************************************
2004
2005 //**SMP multiplication assignment to dense matrices*********************************************
2006 // No special implementation for the SMP multiplication assignment to dense matrices.
2007 //**********************************************************************************************
2008
2009 //**SMP multiplication assignment to sparse matrices********************************************
2010 // No special implementation for the SMP multiplication assignment to sparse matrices.
2011 //**********************************************************************************************
2012
2013 //**Compile time checks*************************************************************************
2022 //**********************************************************************************************
2023};
2024//*************************************************************************************************
2025
2026
2027
2028
2029//=================================================================================================
2030//
2031// GLOBAL BINARY ARITHMETIC OPERATORS
2032//
2033//=================================================================================================
2034
2035//*************************************************************************************************
2048template< typename MT1 // Type of the left-hand side dense matrix
2049 , typename MT2 // Type of the right-hand side sparse matrix
2050 , DisableIf_t< ( IsIdentity_v<MT2> &&
2051 IsSame_v< ElementType_t<MT1>, ElementType_t<MT2> > ) ||
2052 IsZero_v<MT2> >* = nullptr >
2053inline const TDMatSMatMultExpr<MT1,MT2,false,false,false,false>
2054 tdmatsmatmult( const DenseMatrix<MT1,true>& lhs, const SparseMatrix<MT2,false>& rhs )
2055{
2057
2058 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == (*rhs).rows(), "Invalid matrix sizes" );
2059
2060 return TDMatSMatMultExpr<MT1,MT2,false,false,false,false>( *lhs, *rhs );
2061}
2063//*************************************************************************************************
2064
2065
2066//*************************************************************************************************
2080template< typename MT1 // Type of the left-hand side dense matrix
2081 , typename MT2 // Type of the right-hand side sparse matrix
2082 , EnableIf_t< IsIdentity_v<MT2> &&
2083 IsSame_v< ElementType_t<MT1>, ElementType_t<MT2> > >* = nullptr >
2084inline const MT1&
2085 tdmatsmatmult( const DenseMatrix<MT1,true>& lhs, const SparseMatrix<MT2,false>& rhs )
2086{
2088
2089 MAYBE_UNUSED( rhs );
2090
2091 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == (*rhs).rows(), "Invalid matrix sizes" );
2092
2093 return (*lhs);
2094}
2096//*************************************************************************************************
2097
2098
2099//*************************************************************************************************
2112template< typename MT1 // Type of the left-hand side dense matrix
2113 , typename MT2 // Type of the right-hand side sparse matrix
2114 , EnableIf_t< IsZero_v<MT2> >* = nullptr >
2115inline decltype(auto)
2116 tdmatsmatmult( const DenseMatrix<MT1,true>& lhs, const SparseMatrix<MT2,false>& rhs )
2117{
2119
2120 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == (*rhs).rows(), "Invalid matrix sizes" );
2121
2122 using ReturnType = const MultTrait_t< ResultType_t<MT1>, ResultType_t<MT2> >;
2123
2126
2127 return ReturnType( (*lhs).rows(), (*rhs).columns() );
2128}
2130//*************************************************************************************************
2131
2132
2133//*************************************************************************************************
2163template< typename MT1 // Type of the left-hand side dense matrix
2164 , typename MT2 > // Type of the right-hand side sparse matrix
2165inline decltype(auto)
2166 operator*( const DenseMatrix<MT1,true>& lhs, const SparseMatrix<MT2,false>& rhs )
2167{
2169
2170 if( (*lhs).columns() != (*rhs).rows() ) {
2171 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
2172 }
2173
2174 return tdmatsmatmult( *lhs, *rhs );
2175}
2176//*************************************************************************************************
2177
2178
2179
2180
2181//=================================================================================================
2182//
2183// GLOBAL FUNCTIONS
2184//
2185//=================================================================================================
2186
2187//*************************************************************************************************
2212template< typename MT1 // Type of the left-hand side dense matrix
2213 , typename MT2 // Type of the right-hand side dense matrix
2214 , bool SF // Symmetry flag
2215 , bool HF // Hermitian flag
2216 , bool LF // Lower flag
2217 , bool UF > // Upper flag
2218inline decltype(auto) declsym( const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2219{
2221
2222 if( !isSquare( dm ) ) {
2223 BLAZE_THROW_INVALID_ARGUMENT( "Invalid symmetric matrix specification" );
2224 }
2225
2226 using ReturnType = const TDMatSMatMultExpr<MT1,MT2,true,HF,LF,UF>;
2227 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2228}
2230//*************************************************************************************************
2231
2232
2233//*************************************************************************************************
2258template< typename MT1 // Type of the left-hand side dense matrix
2259 , typename MT2 // Type of the right-hand side dense matrix
2260 , bool SF // Symmetry flag
2261 , bool HF // Hermitian flag
2262 , bool LF // Lower flag
2263 , bool UF > // Upper flag
2264inline decltype(auto) declherm( const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2265{
2267
2268 if( !isSquare( dm ) ) {
2269 BLAZE_THROW_INVALID_ARGUMENT( "Invalid Hermitian matrix specification" );
2270 }
2271
2272 using ReturnType = const TDMatSMatMultExpr<MT1,MT2,SF,true,LF,UF>;
2273 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2274}
2276//*************************************************************************************************
2277
2278
2279//*************************************************************************************************
2304template< typename MT1 // Type of the left-hand side dense matrix
2305 , typename MT2 // Type of the right-hand side dense matrix
2306 , bool SF // Symmetry flag
2307 , bool HF // Hermitian flag
2308 , bool LF // Lower flag
2309 , bool UF > // Upper flag
2310inline decltype(auto) decllow( const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2311{
2313
2314 if( !isSquare( dm ) ) {
2315 BLAZE_THROW_INVALID_ARGUMENT( "Invalid lower matrix specification" );
2316 }
2317
2318 using ReturnType = const TDMatSMatMultExpr<MT1,MT2,SF,HF,true,UF>;
2319 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2320}
2322//*************************************************************************************************
2323
2324
2325//*************************************************************************************************
2350template< typename MT1 // Type of the left-hand side dense matrix
2351 , typename MT2 // Type of the right-hand side dense matrix
2352 , bool SF // Symmetry flag
2353 , bool HF // Hermitian flag
2354 , bool UF > // Upper flag
2355inline decltype(auto) declunilow( const TDMatSMatMultExpr<MT1,MT2,SF,HF,false,UF>& dm )
2356{
2358
2359 if( !isSquare( dm ) ) {
2360 BLAZE_THROW_INVALID_ARGUMENT( "Invalid unilower matrix specification" );
2361 }
2362
2363 return declunilow( decllow( *dm ) );
2364}
2366//*************************************************************************************************
2367
2368
2369//*************************************************************************************************
2394template< typename MT1 // Type of the left-hand side dense matrix
2395 , typename MT2 // Type of the right-hand side dense matrix
2396 , bool SF // Symmetry flag
2397 , bool HF // Hermitian flag
2398 , bool UF > // Upper flag
2399inline decltype(auto) declstrlow( const TDMatSMatMultExpr<MT1,MT2,SF,HF,false,UF>& dm )
2400{
2402
2403 if( !isSquare( dm ) ) {
2404 BLAZE_THROW_INVALID_ARGUMENT( "Invalid strictly lower matrix specification" );
2405 }
2406
2407 return declstrlow( decllow( *dm ) );
2408}
2410//*************************************************************************************************
2411
2412
2413//*************************************************************************************************
2438template< typename MT1 // Type of the left-hand side dense matrix
2439 , typename MT2 // Type of the right-hand side dense matrix
2440 , bool SF // Symmetry flag
2441 , bool HF // Hermitian flag
2442 , bool LF // Lower flag
2443 , bool UF > // Upper flag
2444inline decltype(auto) declupp( const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2445{
2447
2448 if( !isSquare( dm ) ) {
2449 BLAZE_THROW_INVALID_ARGUMENT( "Invalid upper matrix specification" );
2450 }
2451
2452 using ReturnType = const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,true>;
2453 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2454}
2456//*************************************************************************************************
2457
2458
2459//*************************************************************************************************
2484template< typename MT1 // Type of the left-hand side dense matrix
2485 , typename MT2 // Type of the right-hand side dense matrix
2486 , bool SF // Symmetry flag
2487 , bool HF // Hermitian flag
2488 , bool LF > // Lower flag
2489inline decltype(auto) decluniupp( const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,false>& dm )
2490{
2492
2493 if( !isSquare( dm ) ) {
2494 BLAZE_THROW_INVALID_ARGUMENT( "Invalid uniupper matrix specification" );
2495 }
2496
2497 return decluniupp( declupp( *dm ) );
2498}
2500//*************************************************************************************************
2501
2502
2503//*************************************************************************************************
2528template< typename MT1 // Type of the left-hand side dense matrix
2529 , typename MT2 // Type of the right-hand side dense matrix
2530 , bool SF // Symmetry flag
2531 , bool HF // Hermitian flag
2532 , bool LF > // Lower flag
2533inline decltype(auto) declstrupp( const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,false>& dm )
2534{
2536
2537 if( !isSquare( dm ) ) {
2538 BLAZE_THROW_INVALID_ARGUMENT( "Invalid strictly upper matrix specification" );
2539 }
2540
2541 return declstrupp( declupp( *dm ) );
2542}
2544//*************************************************************************************************
2545
2546
2547//*************************************************************************************************
2572template< typename MT1 // Type of the left-hand side dense matrix
2573 , typename MT2 // Type of the right-hand side dense matrix
2574 , bool SF // Symmetry flag
2575 , bool HF // Hermitian flag
2576 , bool LF // Lower flag
2577 , bool UF > // Upper flag
2578inline decltype(auto) decldiag( const TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2579{
2581
2582 if( !isSquare( dm ) ) {
2583 BLAZE_THROW_INVALID_ARGUMENT( "Invalid diagonal matrix specification" );
2584 }
2585
2586 using ReturnType = const TDMatSMatMultExpr<MT1,MT2,SF,HF,true,true>;
2587 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2588}
2590//*************************************************************************************************
2591
2592
2593
2594
2595//=================================================================================================
2596//
2597// SIZE SPECIALIZATIONS
2598//
2599//=================================================================================================
2600
2601//*************************************************************************************************
2603template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2604struct Size< TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 0UL >
2605 : public Size<MT1,0UL>
2606{};
2607
2608template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2609struct Size< TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 1UL >
2610 : public Size<MT2,1UL>
2611{};
2613//*************************************************************************************************
2614
2615
2616
2617
2618//=================================================================================================
2619//
2620// ISALIGNED SPECIALIZATIONS
2621//
2622//=================================================================================================
2623
2624//*************************************************************************************************
2626template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2627struct IsAligned< TDMatSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2628 : public IsAligned<MT1>
2629{};
2631//*************************************************************************************************
2632
2633} // namespace blaze
2634
2635#endif
Header file for auxiliary alias declarations.
typename T::CompositeType CompositeType_t
Alias declaration for nested CompositeType type definitions.
Definition: Aliases.h:110
typename T::ResultType ResultType_t
Alias declaration for nested ResultType type definitions.
Definition: Aliases.h:450
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.
Definition: Aliases.h:190
typename T::OppositeType OppositeType_t
Alias declaration for nested OppositeType type definitions.
Definition: Aliases.h:310
typename T::TransposeType TransposeType_t
Alias declaration for nested TransposeType type definitions.
Definition: Aliases.h:550
Header file for run time assertion macros.
Header file for the blaze::checked and blaze::unchecked instances.
Constraints on the storage order of matrix types.
Header file for the conjugate shim.
Header file for the decldiag trait.
Header file for the DeclDiag functor.
Header file for the declherm trait.
Header file for the DeclHerm functor.
Header file for the decllow trait.
Header file for the DeclLow functor.
Header file for the declsym trait.
Header file for the DeclSym functor.
Header file for the declupp trait.
Header file for the DeclUpp functor.
Header file for the EnableIf class template.
Header file for the function trace functionality.
Header file for the If class template.
Header file for the IntegralConstant class template.
Header file for the IsAligned type trait.
Header file for the IsBuiltin type trait.
Header file for the IsColumnMajorMatrix type trait.
Header file for the IsComputation type trait class.
Header file for the isDefault shim.
Header file for the IsDiagonal type trait.
Header file for the IsExpression type trait class.
Header file for the IsIdentity type trait.
Header file for the IsLower type trait.
Header file for the IsResizable type trait.
Header file for the IsStrictlyLower type trait.
Header file for the IsStrictlyUpper type trait.
Header file for the IsSymmetric type trait.
Header file for the IsTriangular type trait.
Header file for the IsUpper type trait.
Header file for the MAYBE_UNUSED function template.
Header file for the multiplication trait.
Header file for the Noop functor.
Header file for the prevMultiple shim.
Constraints on the storage order of matrix types.
Constraint on the data type.
Constraint on the data type.
Base class for dense matrices.
Definition: DenseMatrix.h:82
Base class for sparse matrices.
Definition: SparseMatrix.h:77
Expression object for transpose dense matrix-sparse matrix multiplications.
Definition: TDMatSMatMultExpr.h:130
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TDMatSMatMultExpr.h:249
ElementType_t< ResultType > ElementType
Resulting element type.
Definition: TDMatSMatMultExpr.h:248
static constexpr bool SYM
Flag for symmetric matrices.
Definition: TDMatSMatMultExpr.h:152
static constexpr bool HERM
Flag for Hermitian matrices.
Definition: TDMatSMatMultExpr.h:153
TransposeType_t< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TDMatSMatMultExpr.h:247
static constexpr bool LOW
Flag for lower matrices.
Definition: TDMatSMatMultExpr.h:154
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: TDMatSMatMultExpr.h:344
RightOperand rightOperand() const noexcept
Returns the right-hand side sparse matrix operand.
Definition: TDMatSMatMultExpr.h:390
LeftOperand leftOperand() const noexcept
Returns the left-hand side transpose dense matrix operand.
Definition: TDMatSMatMultExpr.h:380
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TDMatSMatMultExpr.h:434
OppositeType_t< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: TDMatSMatMultExpr.h:246
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: TDMatSMatMultExpr.h:441
ElementType_t< RT1 > ET1
Element type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:135
ElementType_t< RT2 > ET2
Element type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:136
TDMatSMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the TDMatSMatMultExpr class.
Definition: TDMatSMatMultExpr.h:280
If_t< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: TDMatSMatMultExpr.h:262
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TDMatSMatMultExpr.h:402
CompositeType_t< MT1 > CT1
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:137
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: TDMatSMatMultExpr.h:295
bool isAliased(const T *alias) const
Returns whether the expression is aliased with the given address alias.
Definition: TDMatSMatMultExpr.h:414
If_t< IsExpression_v< MT2 >, const MT2, const MT2 & > RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:256
ResultType_t< MT2 > RT2
Result type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:134
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: TDMatSMatMultExpr.h:370
CompositeType_t< MT2 > CT2
Composite type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:138
If_t< IsExpression_v< MT1 >, const MT1, const MT1 & > LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:253
static constexpr bool evaluateRight
Compilation switch for the composite type of the right-hand side sparse matrix expression.
Definition: TDMatSMatMultExpr.h:148
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TDMatSMatMultExpr.h:424
static constexpr bool simdEnabled
Compilation switch for the expression template evaluation strategy.
Definition: TDMatSMatMultExpr.h:267
const ResultType CompositeType
Data type for composite expression templates.
Definition: TDMatSMatMultExpr.h:250
ResultType_t< MT1 > RT1
Result type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:133
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: TDMatSMatMultExpr.h:442
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: TDMatSMatMultExpr.h:360
typename If_t< HERM, DeclHermTrait< MultTrait_t< RT1, RT2 > >, If_t< SYM, DeclSymTrait< MultTrait_t< RT1, RT2 > >, If_t< LOW, If_t< UPP, DeclDiagTrait< MultTrait_t< RT1, RT2 > >, DeclLowTrait< MultTrait_t< RT1, RT2 > > >, If_t< UPP, DeclUppTrait< MultTrait_t< RT1, RT2 > >, MultTrait< RT1, RT2 > > > > >::Type ResultType
Result type for expression template evaluations.
Definition: TDMatSMatMultExpr.h:244
If_t< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: TDMatSMatMultExpr.h:259
static constexpr bool smpAssignable
Compilation switch for the expression template assignment strategy.
Definition: TDMatSMatMultExpr.h:270
static constexpr bool evaluateLeft
Compilation switch for the composite type of the left-hand side dense matrix expression.
Definition: TDMatSMatMultExpr.h:143
static constexpr bool UPP
Flag for upper matrices.
Definition: TDMatSMatMultExpr.h:155
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Header file for the Computation base class.
Header file for the DenseMatrix base class.
Header file for the MatMatMultExpr base class.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:137
decltype(auto) min(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise minimum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1339
decltype(auto) max(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise maximum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1375
decltype(auto) declstrupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as strictly upper.
Definition: DMatDeclStrUppExpr.h:1003
decltype(auto) conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition: DMatMapExpr.h:1464
decltype(auto) decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as diagonal.
Definition: DMatDeclDiagExpr.h:978
decltype(auto) trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:766
decltype(auto) declstrlow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as strictly lower.
Definition: DMatDeclStrLowExpr.h:1003
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:812
decltype(auto) declupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as upper.
Definition: DMatDeclUppExpr.h:1004
decltype(auto) decllow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as lower.
Definition: DMatDeclLowExpr.h:1004
decltype(auto) decluniupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as uniupper.
Definition: DMatDeclUniUppExpr.h:1005
decltype(auto) declherm(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as Hermitian.
Definition: DMatDeclHermExpr.h:1005
decltype(auto) declsym(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as symmetric.
Definition: DMatDeclSymExpr.h:1005
decltype(auto) declunilow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as unilower.
Definition: DMatDeclUniLowExpr.h:1004
bool isDefault(const DiagonalMatrix< MT, SO, DF > &m)
Returns whether the given diagonal matrix is in default state.
Definition: DiagonalMatrix.h:169
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.
Definition: Symmetric.h:79
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition: RowMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.
Definition: StorageOrder.h:84
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.
Definition: RequiresEvaluation.h:81
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.
Definition: MatMatMultExpr.h:103
#define BLAZE_CONSTRAINT_MUST_NOT_BE_ZERO_TYPE(T)
Constraint on the data type.
Definition: Zero.h:81
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.
Definition: DenseMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.
Definition: SparseMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition: ColumnMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_ZERO_TYPE(T)
Constraint on the data type.
Definition: Zero.h:61
BLAZE_ALWAYS_INLINE constexpr auto prevMultiple(T1 value, T2 factor) noexcept
Rounds down an integral value to the previous multiple of a given factor.
Definition: PrevMultiple.h:68
constexpr void reset(Matrix< MT, SO > &matrix)
Resetting the given matrix.
Definition: Matrix.h:806
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:584
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:518
bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition: Matrix.h:1383
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:137
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.
Definition: Assert.h:101
auto smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:162
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
auto smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:194
auto smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:131
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:158
constexpr void MAYBE_UNUSED(const Args &...)
Suppression of unused parameter warnings.
Definition: MaybeUnused.h:81
typename If< Condition >::template Type< T1, T2 > If_t
Auxiliary alias template for the If class template.
Definition: If.h:108
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.
Definition: Exception.h:331
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.
Definition: Exception.h:235
typename EnableIf<!Condition, T >::Type DisableIf_t
Auxiliary type for the EnableIf class template.
Definition: EnableIf.h:175
#define BLAZE_FUNCTION_TRACE
Function trace macro.
Definition: FunctionTrace.h:94
constexpr Unchecked unchecked
Global Unchecked instance.
Definition: Check.h:146
Header file for the exception macros of the math module.
Constraints on the storage order of matrix types.
Header file for all forward declarations for expression class templates.
Header file for the Size type trait.
Header file for the reset shim.
Header file for the serial shim.
Base class for all compute expression templates.
Definition: Computation.h:68
Base template for the DeclDiagTrait class.
Definition: DeclDiagTrait.h:127
Generic wrapper for the decldiag() function.
Definition: DeclDiag.h:61
Base template for the DeclHermTrait class.
Definition: DeclHermTrait.h:126
Generic wrapper for the declherm() function.
Definition: DeclHerm.h:61
Base template for the DeclLowTrait class.
Definition: DeclLowTrait.h:126
Generic wrapper for the decllow() function.
Definition: DeclLow.h:61
Base template for the DeclSymTrait class.
Definition: DeclSymTrait.h:126
Generic wrapper for the declsym() function.
Definition: DeclSym.h:61
Base template for the DeclUppTrait class.
Definition: DeclUppTrait.h:126
Generic wrapper for the declupp() function.
Definition: DeclUpp.h:61
Base class for all matrix/matrix multiplication expression templates.
Definition: MatMatMultExpr.h:71
Base template for the MultTrait class.
Definition: MultTrait.h:130
Generic wrapper for the null function.
Definition: Noop.h:62
System settings for performance optimizations.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
Header file for the IsZero type trait.
Header file for the RequiresEvaluation type trait.
Header file for basic type definitions.
Header file for the generic max algorithm.
Header file for the generic min algorithm.