Blaze 3.9
TSMatTDMatMultExpr.h
Go to the documentation of this file.
1//=================================================================================================
33//=================================================================================================
34
35#ifndef _BLAZE_MATH_EXPRESSIONS_TSMATTDMATMULTEXPR_H_
36#define _BLAZE_MATH_EXPRESSIONS_TSMATTDMATMULTEXPR_H_
37
38
39//*************************************************************************************************
40// Includes
41//*************************************************************************************************
42
43#include <blaze/math/Aliases.h>
97#include <blaze/util/Assert.h>
98#include <blaze/util/EnableIf.h>
102#include <blaze/util/mpl/If.h>
103#include <blaze/util/Types.h>
105
106
107namespace blaze {
108
109//=================================================================================================
110//
111// CLASS SMATDMATMULTEXPR
112//
113//=================================================================================================
114
115//*************************************************************************************************
122template< typename MT1 // Type of the left-hand side dense matrix
123 , typename MT2 // Type of the right-hand side sparse matrix
124 , bool SF // Symmetry flag
125 , bool HF // Hermitian flag
126 , bool LF // Lower flag
127 , bool UF > // Upper flag
129 : public MatMatMultExpr< DenseMatrix< TSMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, true > >
130 , private Computation
131{
132 private:
133 //**Type definitions****************************************************************************
140 //**********************************************************************************************
141
142 //**********************************************************************************************
144 static constexpr bool evaluateLeft = ( IsComputation_v<MT1> || RequiresEvaluation_v<MT1> );
145 //**********************************************************************************************
146
147 //**********************************************************************************************
149 static constexpr bool evaluateRight = ( IsComputation_v<MT2> || RequiresEvaluation_v<MT2> );
150 //**********************************************************************************************
151
152 //**********************************************************************************************
153 static constexpr bool SYM = ( SF && !( HF || LF || UF ) );
154 static constexpr bool HERM = ( HF && !( LF || UF ) );
155 static constexpr bool LOW = ( LF || ( ( SF || HF ) && UF ) );
156 static constexpr bool UPP = ( UF || ( ( SF || HF ) && LF ) );
157 //**********************************************************************************************
158
159 //**********************************************************************************************
161
166 template< typename T1, typename T2, typename T3 >
167 static constexpr bool CanExploitSymmetry_v = ( IsSymmetric_v<T2> || IsSymmetric_v<T3> );
169 //**********************************************************************************************
170
171 //**********************************************************************************************
173
177 template< typename T1, typename T2, typename T3 >
178 static constexpr bool IsEvaluationRequired_v =
179 ( ( evaluateLeft || evaluateRight ) && !CanExploitSymmetry_v<T1,T2,T3> );
181 //**********************************************************************************************
182
183 //**********************************************************************************************
185
188 template< typename T1, typename T2, typename T3 >
189 static constexpr bool UseOptimizedKernel_v =
190 ( useOptimizedKernels &&
191 !IsDiagonal_v<T3> &&
192 !IsResizable_v< ElementType_t<T1> > &&
193 !IsResizable_v<ET1> );
195 //**********************************************************************************************
196
197 //**********************************************************************************************
199
202 template< typename T1, typename T2, typename T3 >
203 static constexpr bool UseDefaultKernel_v = !UseOptimizedKernel_v<T1,T2,T3>;
205 //**********************************************************************************************
206
207 //**********************************************************************************************
209
212 using ForwardFunctor = If_t< HERM
213 , DeclHerm
214 , If_t< SYM
215 , DeclSym
216 , If_t< LOW
217 , If_t< UPP
218 , DeclDiag
219 , DeclLow >
220 , If_t< UPP
221 , DeclUpp
222 , Noop > > > >;
224 //**********************************************************************************************
225
226 public:
227 //**Type definitions****************************************************************************
230
233
235 using ResultType = typename If_t< HERM
237 , If_t< SYM
239 , If_t< LOW
240 , If_t< UPP
243 , If_t< UPP
245 , MultTrait<RT1,RT2> > > > >::Type;
246
250 using ReturnType = const ElementType;
251 using CompositeType = const ResultType;
252
254 using LeftOperand = If_t< IsExpression_v<MT1>, const MT1, const MT1& >;
255
257 using RightOperand = If_t< IsExpression_v<MT2>, const MT2, const MT2& >;
258
261
264 //**********************************************************************************************
265
266 //**Compilation flags***************************************************************************
268 static constexpr bool simdEnabled = false;
269
271 static constexpr bool smpAssignable =
272 ( !evaluateLeft && MT1::smpAssignable && !evaluateRight && MT2::smpAssignable );
273 //**********************************************************************************************
274
275 //**Constructor*********************************************************************************
281 inline TSMatTDMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
282 : lhs_( lhs ) // Left-hand side sparse matrix of the multiplication expression
283 , rhs_( rhs ) // Right-hand side dense matrix of the multiplication expression
284 {
285 BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
286 }
287 //**********************************************************************************************
288
289 //**Access operator*****************************************************************************
296 inline ReturnType operator()( size_t i, size_t j ) const {
297 BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
298 BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
299
300 if( IsDiagonal_v<MT1> ) {
301 return lhs_(i,i) * rhs_(i,j);
302 }
303 else if( IsDiagonal_v<MT2> ) {
304 return lhs_(i,j) * rhs_(j,j);
305 }
306 else if( IsTriangular_v<MT1> || IsTriangular_v<MT2> ) {
307 const size_t begin( ( IsUpper_v<MT1> )
308 ?( ( IsLower_v<MT2> )
309 ?( max( ( IsStrictlyUpper_v<MT1> ? i+1UL : i )
310 , ( IsStrictlyLower_v<MT2> ? j+1UL : j ) ) )
311 :( IsStrictlyUpper_v<MT1> ? i+1UL : i ) )
312 :( ( IsLower_v<MT2> )
313 ?( IsStrictlyLower_v<MT2> ? j+1UL : j )
314 :( 0UL ) ) );
315 const size_t end( ( IsLower_v<MT1> )
316 ?( ( IsUpper_v<MT2> )
317 ?( min( ( IsStrictlyLower_v<MT1> ? i : i+1UL )
318 , ( IsStrictlyUpper_v<MT2> ? j : j+1UL ) ) )
319 :( IsStrictlyLower_v<MT1> ? i : i+1UL ) )
320 :( ( IsUpper_v<MT2> )
321 ?( IsStrictlyUpper_v<MT2> ? j : j+1UL )
322 :( lhs_.columns() ) ) );
323
324 if( begin >= end ) return ElementType();
325
326 const size_t n( end - begin );
327
328 return subvector( row( lhs_, i, unchecked ), begin, n, unchecked ) *
330 }
331 else {
332 return row( lhs_, i, unchecked ) * column( rhs_, j, unchecked );
333 }
334 }
335 //**********************************************************************************************
336
337 //**At function*********************************************************************************
345 inline ReturnType at( size_t i, size_t j ) const {
346 if( i >= lhs_.rows() ) {
347 BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
348 }
349 if( j >= rhs_.columns() ) {
350 BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
351 }
352 return (*this)(i,j);
353 }
354 //**********************************************************************************************
355
356 //**Rows function*******************************************************************************
361 inline size_t rows() const noexcept {
362 return lhs_.rows();
363 }
364 //**********************************************************************************************
365
366 //**Columns function****************************************************************************
371 inline size_t columns() const noexcept {
372 return rhs_.columns();
373 }
374 //**********************************************************************************************
375
376 //**Left operand access*************************************************************************
381 inline LeftOperand leftOperand() const noexcept {
382 return lhs_;
383 }
384 //**********************************************************************************************
385
386 //**Right operand access************************************************************************
391 inline RightOperand rightOperand() const noexcept {
392 return rhs_;
393 }
394 //**********************************************************************************************
395
396 //**********************************************************************************************
402 template< typename T >
403 inline bool canAlias( const T* alias ) const noexcept {
404 return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
405 }
406 //**********************************************************************************************
407
408 //**********************************************************************************************
414 template< typename T >
415 inline bool isAliased( const T* alias ) const noexcept {
416 return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
417 }
418 //**********************************************************************************************
419
420 //**********************************************************************************************
425 inline bool isAligned() const noexcept {
426 return rhs_.isAligned();
427 }
428 //**********************************************************************************************
429
430 //**********************************************************************************************
435 inline bool canSMPAssign() const noexcept {
436 return ( rows() * columns() >= SMP_TSMATTDMATMULT_THRESHOLD ) && !IsDiagonal_v<MT2>;
437 }
438 //**********************************************************************************************
439
440 private:
441 //**Member variables****************************************************************************
444 //**********************************************************************************************
445
446 //**Assignment to dense matrices****************************************************************
459 template< typename MT // Type of the target dense matrix
460 , bool SO > // Storage order of the target dense matrix
461 friend inline auto assign( DenseMatrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
463 {
465
466 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
467 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
468
469 LT A( serial( rhs.lhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
470 RT B( serial( rhs.rhs_ ) ); // Evaluation of the left-hand side dense matrix operand
471
472 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
473 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
474 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
475 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
476 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
477 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
478
479 TSMatTDMatMultExpr::selectAssignKernel( *lhs, A, B );
480 }
482 //**********************************************************************************************
483
484 //**Assignment to dense matrices (kernel selection)*********************************************
495 template< typename MT3 // Type of the left-hand side target matrix
496 , typename MT4 // Type of the left-hand side matrix operand
497 , typename MT5 > // Type of the right-hand side matrix operand
498 static inline void selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
499 {
500 const size_t size( C.rows() * C.columns() );
501
502 if( ( IsRowMajorMatrix_v<MT3> && size < TSMATTDMATMULT_THRESHOLD ) ||
503 ( IsColumnMajorMatrix_v<MT3> && size < 625UL ) )
504 selectSmallAssignKernel( C, A, B );
505 else
506 selectLargeAssignKernel( C, A, B );
507 }
509 //**********************************************************************************************
510
511 //**Default assignment to dense matrices********************************************************
526 template< typename MT3 // Type of the left-hand side target matrix
527 , typename MT4 // Type of the left-hand side matrix operand
528 , typename MT5 > // Type of the right-hand side matrix operand
529 static inline void selectDefaultAssignKernel( MT3& C, const MT4& A, const MT5& B )
530 {
531 reset( C );
532
533 if( IsDiagonal_v<MT5> )
534 {
535 for( size_t i=0UL; i<A.columns(); ++i )
536 {
537 const auto end( A.end(i) );
538 auto element( A.begin(i) );
539
540 for( ; element!=end; ++element ) {
541 C(element->index(),i) = element->value() * B(i,i);
542 }
543 }
544 }
545 else
546 {
547 const size_t block( 64UL );
548
549 for( size_t jj=0UL; jj<B.columns(); jj+=block )
550 {
551 const size_t jpos( ( jj+block > B.columns() )?( B.columns() ):( jj+block ) );
552
553 for( size_t i=0UL; i<A.columns(); ++i )
554 {
555 const auto end( A.end(i) );
556 auto element( A.begin(i) );
557
558 for( ; element!=end; ++element )
559 {
560 const size_t i1( element->index() );
561
562 const size_t jbegin( ( IsUpper_v<MT5> )
563 ?( ( SYM || HERM || UPP )
564 ?( max( i1, IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) )
565 :( max( IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) ) )
566 :( SYM || HERM || UPP ? max(i1,jj) : jj ) );
567 const size_t jend( ( IsLower_v<MT5> )
568 ?( ( LOW )
569 ?( min( i1+1UL, IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) )
570 :( min( IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) ) )
571 :( LOW ? min(i1+1UL,jpos) : jpos ) );
572
573 if( jbegin >= jend )
574 continue;
575
576 for( size_t j=jbegin; j<jend; ++j ) {
577 if( isDefault( C(i1,j) ) )
578 C(i1,j) = element->value() * B(i,j);
579 else
580 C(i1,j) += element->value() * B(i,j);
581 }
582 }
583 }
584 }
585 }
586
587 if( SYM || HERM ) {
588 for( size_t j=0UL; j<B.columns(); ++j ) {
589 for( size_t i=j+1UL; i<A.rows(); ++i ) {
590 C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
591 }
592 }
593 }
594 }
596 //**********************************************************************************************
597
598 //**Default assignment to dense matrices (small matrices)***************************************
612 template< typename MT3 // Type of the left-hand side target matrix
613 , typename MT4 // Type of the left-hand side matrix operand
614 , typename MT5 > // Type of the right-hand side matrix operand
615 static inline auto selectSmallAssignKernel( MT3& C, const MT4& A, const MT5& B )
616 -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
617 {
618 selectDefaultAssignKernel( C, A, B );
619 }
621 //**********************************************************************************************
622
623 //**Optimized assignment to dense matrices (small matrices)*************************************
638 template< typename MT3 // Type of the left-hand side target matrix
639 , typename MT4 // Type of the left-hand side matrix operand
640 , typename MT5 > // Type of the right-hand side matrix operand
641 static inline auto selectSmallAssignKernel( MT3& C, const MT4& A, const MT5& B )
642 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
643 {
644 const size_t block( IsRowMajorMatrix_v<MT3> ? 128UL : 64UL );
645
646 for( size_t jj=0UL; jj<B.columns(); jj+=block )
647 {
648 const size_t jpos( ( jj+block > B.columns() )?( B.columns() ):( jj+block ) );
649
650 for( size_t i=0UL; i<A.rows(); ++i ) {
651 for( size_t j=jj; j<jpos; ++j ) {
652 reset( C(i,j) );
653 }
654 }
655
656 for( size_t i=0UL; i<A.columns(); ++i )
657 {
658 const auto end( A.end(i) );
659 auto element( A.begin(i) );
660
661 const size_t nonzeros( A.nonZeros(i) );
662 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
663 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
664
665 for( size_t k=0UL; k<kpos; k+=4UL )
666 {
667 const size_t i1( element->index() );
668 const ET1 v1( element->value() );
669 ++element;
670 const size_t i2( element->index() );
671 const ET1 v2( element->value() );
672 ++element;
673 const size_t i3( element->index() );
674 const ET1 v3( element->value() );
675 ++element;
676 const size_t i4( element->index() );
677 const ET1 v4( element->value() );
678 ++element;
679
680 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
681
682 const size_t jbegin( ( IsUpper_v<MT5> )
683 ?( ( SYM || HERM || UPP )
684 ?( max( i1, IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) )
685 :( max( IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) ) )
686 :( SYM || HERM || UPP ? max(i1,jj) : jj ) );
687 const size_t jend( ( IsLower_v<MT5> )
688 ?( ( LOW )
689 ?( min( i4+1UL, IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) )
690 :( min( IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) ) )
691 :( LOW ? min(i4+1UL,jpos) : jpos ) );
692
693 if( jbegin >= jend )
694 continue;
695
696 for( size_t j=jbegin; j<jend; ++j ) {
697 C(i1,j) += v1 * B(i,j);
698 C(i2,j) += v2 * B(i,j);
699 C(i3,j) += v3 * B(i,j);
700 C(i4,j) += v4 * B(i,j);
701 }
702 }
703
704 for( ; element!=end; ++element )
705 {
706 const size_t i1( element->index() );
707
708 const size_t jbegin( ( IsUpper_v<MT5> )
709 ?( ( SYM || HERM || UPP )
710 ?( max( i1, IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) )
711 :( max( IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) ) )
712 :( SYM || HERM || UPP ? max(i1,jj) : jj ) );
713 const size_t jend( ( IsLower_v<MT5> )
714 ?( ( LOW )
715 ?( min( i1+1UL, IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) )
716 :( min( IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) ) )
717 :( LOW ? min(i1+1UL,jpos) : jpos ) );
718
719 if( jbegin >= jend )
720 continue;
721
722 for( size_t j=jbegin; j<jend; ++j ) {
723 C(i1,j) += element->value() * B(i,j);
724 }
725 }
726 }
727 }
728
729 if( SYM || HERM ) {
730 for( size_t j=0UL; j<B.columns(); ++j ) {
731 for( size_t i=j+1UL; i<A.rows(); ++i ) {
732 C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
733 }
734 }
735 }
736 }
738 //**********************************************************************************************
739
740 //**Default assignment to dense matrices (large matrices)***************************************
754 template< typename MT3 // Type of the left-hand side target matrix
755 , typename MT4 // Type of the left-hand side matrix operand
756 , typename MT5 > // Type of the right-hand side matrix operand
757 static inline auto selectLargeAssignKernel( MT3& C, const MT4& A, const MT5& B )
758 -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
759 {
760 selectDefaultAssignKernel( C, A, B );
761 }
763 //**********************************************************************************************
764
765 //**Optimized assignment to dense matrices (large matrices)*************************************
780 template< typename MT3 // Type of the left-hand side target matrix
781 , typename MT4 // Type of the left-hand side matrix operand
782 , typename MT5 > // Type of the right-hand side matrix operand
783 static inline auto selectLargeAssignKernel( MT3& C, const MT4& A, const MT5& B )
784 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
785 {
787
788 const ForwardFunctor fwd;
789
790 const OppositeType_t<MT4> tmp( serial( A ) );
791 assign( C, fwd( tmp * B ) );
792 }
794 //**********************************************************************************************
795
796 //**Assignment to sparse matrices***************************************************************
809 template< typename MT // Type of the target sparse matrix
810 , bool SO > // Storage order of the target sparse matrix
811 friend inline auto assign( SparseMatrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
812 -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
813 {
815
816 using TmpType = If_t< SO, ResultType, OppositeType >;
817
824
825 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
826 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
827
828 const ForwardFunctor fwd;
829
830 const TmpType tmp( serial( rhs ) );
831 assign( *lhs, fwd( tmp ) );
832 }
834 //**********************************************************************************************
835
836 //**Restructuring assignment********************************************************************
851 template< typename MT // Type of the target matrix
852 , bool SO > // Storage order of the target matrix
853 friend inline auto assign( Matrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
854 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
855 {
857
859
860 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
861 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
862
863 const ForwardFunctor fwd;
864
865 decltype(auto) A( transIf< IsSymmetric_v<MT1> >( rhs.lhs_ ) );
866 decltype(auto) B( transIf< IsSymmetric_v<MT2> >( rhs.rhs_ ) );
867
868 assign( *lhs, fwd( A * B ) );
869 }
871 //**********************************************************************************************
872
873 //**Addition assignment to dense matrices*******************************************************
886 template< typename MT // Type of the target dense matrix
887 , bool SO > // Storage order of the target dense matrix
888 friend inline auto addAssign( DenseMatrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
889 -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
890 {
892
893 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
894 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
895
896 LT A( serial( rhs.lhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
897 RT B( serial( rhs.rhs_ ) ); // Evaluation of the left-hand side dense matrix operand
898
899 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
900 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
901 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
902 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
903 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
904 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
905
906 TSMatTDMatMultExpr::selectAddAssignKernel( *lhs, A, B );
907 }
909 //**********************************************************************************************
910
911 //**Addition assignment to dense matrices (kernel selection)************************************
922 template< typename MT3 // Type of the left-hand side target matrix
923 , typename MT4 // Type of the left-hand side matrix operand
924 , typename MT5 > // Type of the right-hand side matrix operand
925 static inline void selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
926 {
927 const size_t size( C.rows() * C.columns() );
928
929 if( ( IsRowMajorMatrix_v<MT3> && size < TSMATTDMATMULT_THRESHOLD ) ||
930 ( IsColumnMajorMatrix_v<MT3> && size < 625UL ) )
931 selectSmallAddAssignKernel( C, A, B );
932 else
933 selectLargeAddAssignKernel( C, A, B );
934 }
936 //**********************************************************************************************
937
938 //**Default addition assignment to dense matrices***********************************************
953 template< typename MT3 // Type of the left-hand side target matrix
954 , typename MT4 // Type of the left-hand side matrix operand
955 , typename MT5 > // Type of the right-hand side matrix operand
956 static inline void selectDefaultAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
957 {
958 if( IsDiagonal_v<MT5> )
959 {
960 for( size_t i=0UL; i<A.columns(); ++i )
961 {
962 const auto end( A.end(i) );
963 auto element( A.begin(i) );
964
965 for( ; element!=end; ++element ) {
966 C(element->index(),i) += element->value() * B(i,i);
967 }
968 }
969 }
970 else
971 {
972 const size_t block( 64UL );
973
974 for( size_t jj=0UL; jj<B.columns(); jj+=block )
975 {
976 const size_t jpos( ( jj+block > B.columns() )?( B.columns() ):( jj+block ) );
977
978 for( size_t i=0UL; i<A.columns(); ++i )
979 {
980 const auto end( A.end(i) );
981 auto element( A.begin(i) );
982
983 for( ; element!=end; ++element )
984 {
985 const size_t i1( element->index() );
986
987 const size_t jbegin( ( IsUpper_v<MT5> )
988 ?( ( UPP )
989 ?( max( i1, IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) )
990 :( max( IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) ) )
991 :( UPP ? max(i1,jj) : jj ) );
992 const size_t jend( ( IsLower_v<MT5> )
993 ?( ( LOW )
994 ?( min( i1+1UL, IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) )
995 :( min( IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) ) )
996 :( LOW ? min(i1+1UL,jpos) : jpos ) );
997
998 if( jbegin >= jend )
999 continue;
1000
1001 for( size_t j=jbegin; j<jend; ++j ) {
1002 C(i1,j) += element->value() * B(i,j);
1003 }
1004 }
1005 }
1006 }
1007 }
1008 }
1010 //**********************************************************************************************
1011
1012 //**Default addition assignment to dense matrices (small matrices)******************************
1026 template< typename MT3 // Type of the left-hand side target matrix
1027 , typename MT4 // Type of the left-hand side matrix operand
1028 , typename MT5 > // Type of the right-hand side matrix operand
1029 static inline auto selectSmallAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1030 -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
1031 {
1032 selectDefaultAddAssignKernel( C, A, B );
1033 }
1035 //**********************************************************************************************
1036
1037 //**Optimized addition assignment to dense matrices (small matrices)****************************
1052 template< typename MT3 // Type of the left-hand side target matrix
1053 , typename MT4 // Type of the left-hand side matrix operand
1054 , typename MT5 > // Type of the right-hand side matrix operand
1055 static inline auto selectSmallAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1056 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1057 {
1058 const size_t block( IsRowMajorMatrix_v<MT3> ? 128UL : 64UL );
1059
1060 for( size_t jj=0UL; jj<B.columns(); jj+=block )
1061 {
1062 const size_t jpos( ( jj+block > B.columns() )?( B.columns() ):( jj+block ) );
1063
1064 for( size_t i=0UL; i<A.columns(); ++i )
1065 {
1066 const auto end( A.end(i) );
1067 auto element( A.begin(i) );
1068
1069 const size_t nonzeros( A.nonZeros(i) );
1070 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1071 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1072
1073 for( size_t k=0UL; k<kpos; k+=4UL )
1074 {
1075 const size_t i1( element->index() );
1076 const ET1 v1( element->value() );
1077 ++element;
1078 const size_t i2( element->index() );
1079 const ET1 v2( element->value() );
1080 ++element;
1081 const size_t i3( element->index() );
1082 const ET1 v3( element->value() );
1083 ++element;
1084 const size_t i4( element->index() );
1085 const ET1 v4( element->value() );
1086 ++element;
1087
1088 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1089
1090 const size_t jbegin( ( IsUpper_v<MT5> )
1091 ?( ( UPP )
1092 ?( max( i1, IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) )
1093 :( max( IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) ) )
1094 :( UPP ? max(i1,jj) : jj ) );
1095 const size_t jend( ( IsLower_v<MT5> )
1096 ?( ( LOW )
1097 ?( min( i4+1UL, IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) )
1098 :( min( IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) ) )
1099 :( LOW ? min(i4+1UL,jpos) : jpos ) );
1100
1101 if( jbegin >= jend )
1102 continue;
1103
1104 for( size_t j=jbegin; j<jend; ++j ) {
1105 C(i1,j) += v1 * B(i,j);
1106 C(i2,j) += v2 * B(i,j);
1107 C(i3,j) += v3 * B(i,j);
1108 C(i4,j) += v4 * B(i,j);
1109 }
1110 }
1111
1112 for( ; element!=end; ++element )
1113 {
1114 const size_t i1( element->index() );
1115
1116 const size_t jbegin( ( IsUpper_v<MT5> )
1117 ?( ( UPP )
1118 ?( max( i1, IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) )
1119 :( max( IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) ) )
1120 :( UPP ? max(i1,jj) : jj ) );
1121 const size_t jend( ( IsLower_v<MT5> )
1122 ?( ( LOW )
1123 ?( min( i1+1UL, IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) )
1124 :( min( IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) ) )
1125 :( LOW ? min(i1+1UL,jpos) : jpos ) );
1126
1127 if( jbegin >= jend )
1128 continue;
1129
1130 for( size_t j=jbegin; j<jend; ++j ) {
1131 C(i1,j) += element->value() * B(i,j);
1132 }
1133 }
1134 }
1135 }
1136 }
1138 //**********************************************************************************************
1139
1140 //**Default addition assignment to dense matrices (large matrices)******************************
1154 template< typename MT3 // Type of the left-hand side target matrix
1155 , typename MT4 // Type of the left-hand side matrix operand
1156 , typename MT5 > // Type of the right-hand side matrix operand
1157 static inline auto selectLargeAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1158 -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
1159 {
1160 selectDefaultAddAssignKernel( C, A, B );
1161 }
1163 //**********************************************************************************************
1164
1165 //**Optimized addition assignment to dense matrices (large matrices)****************************
1180 template< typename MT3 // Type of the left-hand side target matrix
1181 , typename MT4 // Type of the left-hand side matrix operand
1182 , typename MT5 > // Type of the right-hand side matrix operand
1183 static inline auto selectLargeAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
1184 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1185 {
1187
1188 const ForwardFunctor fwd;
1189
1190 const OppositeType_t<MT4> tmp( serial( A ) );
1191 addAssign( C, fwd( tmp * B ) );
1192 }
1194 //**********************************************************************************************
1195
1196 //**Restructuring addition assignment***********************************************************
1211 template< typename MT // Type of the target matrix
1212 , bool SO > // Storage order of the target matrix
1213 friend inline auto addAssign( Matrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
1214 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1215 {
1217
1219
1220 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1221 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1222
1223 const ForwardFunctor fwd;
1224
1225 decltype(auto) A( transIf< IsSymmetric_v<MT1> >( rhs.lhs_ ) );
1226 decltype(auto) B( transIf< IsSymmetric_v<MT2> >( rhs.rhs_ ) );
1227
1228 addAssign( *lhs, fwd( A * B ) );
1229 }
1231 //**********************************************************************************************
1232
1233 //**Addition assignment to sparse matrices******************************************************
1234 // No special implementation for the addition assignment to sparse matrices.
1235 //**********************************************************************************************
1236
1237 //**Subtraction assignment to dense matrices****************************************************
1250 template< typename MT // Type of the target dense matrix
1251 , bool SO > // Storage order of the target dense matrix
1252 friend inline auto subAssign( DenseMatrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
1253 -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1254 {
1256
1257 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1258 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1259
1260 LT A( serial( rhs.lhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1261 RT B( serial( rhs.rhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1262
1263 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1264 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1265 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1266 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1267 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1268 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
1269
1270 TSMatTDMatMultExpr::selectSubAssignKernel( *lhs, A, B );
1271 }
1273 //**********************************************************************************************
1274
1275 //**Subtraction assignment to dense matrices (kernel selection)*********************************
1286 template< typename MT3 // Type of the left-hand side target matrix
1287 , typename MT4 // Type of the left-hand side matrix operand
1288 , typename MT5 > // Type of the right-hand side matrix operand
1289 static inline void selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1290 {
1291 const size_t size( C.rows() * C.columns() );
1292
1293 if( ( IsRowMajorMatrix_v<MT3> && size < TSMATTDMATMULT_THRESHOLD ) ||
1294 ( IsColumnMajorMatrix_v<MT3> && size < 625UL ) )
1295 selectSmallSubAssignKernel( C, A, B );
1296 else
1297 selectLargeSubAssignKernel( C, A, B );
1298 }
1300 //**********************************************************************************************
1301
1302 //**Default subtraction assignment to dense matrices********************************************
1317 template< typename MT3 // Type of the left-hand side target matrix
1318 , typename MT4 // Type of the left-hand side matrix operand
1319 , typename MT5 > // Type of the right-hand side matrix operand
1320 static inline void selectDefaultSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1321 {
1322 if( IsDiagonal_v<MT5> )
1323 {
1324 for( size_t i=0UL; i<A.columns(); ++i )
1325 {
1326 const auto end( A.end(i) );
1327 auto element( A.begin(i) );
1328
1329 for( ; element!=end; ++element ) {
1330 C(element->index(),i) -= element->value() * B(i,i);
1331 }
1332 }
1333 }
1334 else
1335 {
1336 const size_t block( 64UL );
1337
1338 for( size_t jj=0UL; jj<B.columns(); jj+=block )
1339 {
1340 const size_t jpos( ( jj+block > B.columns() )?( B.columns() ):( jj+block ) );
1341
1342 for( size_t i=0UL; i<A.columns(); ++i )
1343 {
1344 const auto end( A.end(i) );
1345 auto element( A.begin(i) );
1346
1347 for( ; element!=end; ++element )
1348 {
1349 const size_t i1( element->index() );
1350
1351 const size_t jbegin( ( IsUpper_v<MT5> )
1352 ?( ( UPP )
1353 ?( max( i1, IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) )
1354 :( max( IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) ) )
1355 :( UPP ? max(i1,jj) : jj ) );
1356 const size_t jend( ( IsLower_v<MT5> )
1357 ?( ( LOW )
1358 ?( min( i1+1UL, IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) )
1359 :( min( IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) ) )
1360 :( LOW ? min(i1+1UL,jpos) : jpos ) );
1361
1362 if( jbegin >= jend )
1363 continue;
1364
1365 for( size_t j=jbegin; j<jend; ++j ) {
1366 C(i1,j) -= element->value() * B(i,j);
1367 }
1368 }
1369 }
1370 }
1371 }
1372 }
1374 //**********************************************************************************************
1375
1376 //**Default subtraction assignment to dense matrices (small matrices)***************************
1390 template< typename MT3 // Type of the left-hand side target matrix
1391 , typename MT4 // Type of the left-hand side matrix operand
1392 , typename MT5 > // Type of the right-hand side matrix operand
1393 static inline auto selectSmallSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1394 -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
1395 {
1396 selectDefaultSubAssignKernel( C, A, B );
1397 }
1399 //**********************************************************************************************
1400
1401 //**Optimized subtraction assignment to dense matrices (small matrices)*************************
1416 template< typename MT3 // Type of the left-hand side target matrix
1417 , typename MT4 // Type of the left-hand side matrix operand
1418 , typename MT5 > // Type of the right-hand side matrix operand
1419 static inline auto selectSmallSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1420 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1421 {
1422 const size_t block( IsRowMajorMatrix_v<MT3> ? 128UL : 64UL );
1423
1424 for( size_t jj=0UL; jj<B.columns(); jj+=block )
1425 {
1426 const size_t jpos( ( jj+block > B.columns() )?( B.columns() ):( jj+block ) );
1427
1428 for( size_t i=0UL; i<A.columns(); ++i )
1429 {
1430 const auto end( A.end(i) );
1431 auto element( A.begin(i) );
1432
1433 const size_t nonzeros( A.nonZeros(i) );
1434 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1435 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1436
1437 for( size_t k=0UL; k<kpos; k+=4UL )
1438 {
1439 const size_t i1( element->index() );
1440 const ET1 v1( element->value() );
1441 ++element;
1442 const size_t i2( element->index() );
1443 const ET1 v2( element->value() );
1444 ++element;
1445 const size_t i3( element->index() );
1446 const ET1 v3( element->value() );
1447 ++element;
1448 const size_t i4( element->index() );
1449 const ET1 v4( element->value() );
1450 ++element;
1451
1452 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1453
1454 const size_t jbegin( ( IsUpper_v<MT5> )
1455 ?( ( UPP )
1456 ?( max( i1, IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) )
1457 :( max( IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) ) )
1458 :( UPP ? max(i1,jj) : jj ) );
1459 const size_t jend( ( IsLower_v<MT5> )
1460 ?( ( LOW )
1461 ?( min( i4+1UL, IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) )
1462 :( min( IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) ) )
1463 :( LOW ? min(i4+1UL,jpos) : jpos ) );
1464
1465 if( jbegin >= jend )
1466 continue;
1467
1468 for( size_t j=jbegin; j<jend; ++j ) {
1469 C(i1,j) -= v1 * B(i,j);
1470 C(i2,j) -= v2 * B(i,j);
1471 C(i3,j) -= v3 * B(i,j);
1472 C(i4,j) -= v4 * B(i,j);
1473 }
1474 }
1475
1476 for( ; element!=end; ++element )
1477 {
1478 const size_t i1( element->index() );
1479
1480 const size_t jbegin( ( IsUpper_v<MT5> )
1481 ?( ( UPP )
1482 ?( max( i1, IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) )
1483 :( max( IsStrictlyUpper_v<MT5> ? i+1UL : i, jj ) ) )
1484 :( UPP ? max(i1,jj) : jj ) );
1485 const size_t jend( ( IsLower_v<MT5> )
1486 ?( ( LOW )
1487 ?( min( i1+1UL, IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) )
1488 :( min( IsStrictlyLower_v<MT5> ? i : i+1UL, jpos ) ) )
1489 :( LOW ? min(i1+1UL,jpos) : jpos ) );
1490
1491 if( jbegin >= jend )
1492 continue;
1493
1494 for( size_t j=jbegin; j<jend; ++j ) {
1495 C(i1,j) -= element->value() * B(i,j);
1496 }
1497 }
1498 }
1499 }
1500 }
1502 //**********************************************************************************************
1503
1504 //**Default subtraction assignment to dense matrices (large matrices)***************************
1518 template< typename MT3 // Type of the left-hand side target matrix
1519 , typename MT4 // Type of the left-hand side matrix operand
1520 , typename MT5 > // Type of the right-hand side matrix operand
1521 static inline auto selectLargeSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1522 -> EnableIf_t< UseDefaultKernel_v<MT3,MT4,MT5> >
1523 {
1524 selectDefaultSubAssignKernel( C, A, B );
1525 }
1527 //**********************************************************************************************
1528
1529 //**Optimized subtraction assignment to dense matrices (large matrices)*************************
1544 template< typename MT3 // Type of the left-hand side target matrix
1545 , typename MT4 // Type of the left-hand side matrix operand
1546 , typename MT5 > // Type of the right-hand side matrix operand
1547 static inline auto selectLargeSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1548 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1549 {
1551
1552 const ForwardFunctor fwd;
1553
1554 const OppositeType_t<MT4> tmp( serial( A ) );
1555 subAssign( C, fwd( tmp * B ) );
1556 }
1558 //**********************************************************************************************
1559
1560 //**Restructuring subtraction assignment********************************************************
1575 template< typename MT // Type of the target matrix
1576 , bool SO > // Storage order of the target matrix
1577 friend inline auto subAssign( Matrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
1578 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1579 {
1581
1583
1584 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1585 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1586
1587 const ForwardFunctor fwd;
1588
1589 decltype(auto) A( transIf< IsSymmetric_v<MT1> >( rhs.lhs_ ) );
1590 decltype(auto) B( transIf< IsSymmetric_v<MT2> >( rhs.rhs_ ) );
1591
1592 subAssign( *lhs, fwd( A * B ) );
1593 }
1595 //**********************************************************************************************
1596
1597 //**Subtraction assignment to sparse matrices***************************************************
1598 // No special implementation for the subtraction assignment to sparse matrices.
1599 //**********************************************************************************************
1600
1601 //**Schur product assignment to dense matrices**************************************************
1614 template< typename MT // Type of the target dense matrix
1615 , bool SO > // Storage order of the target dense matrix
1616 friend inline void schurAssign( DenseMatrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
1617 {
1619
1623
1624 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1625 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1626
1627 const ResultType tmp( serial( rhs ) );
1628 schurAssign( *lhs, tmp );
1629 }
1631 //**********************************************************************************************
1632
1633 //**Schur product assignment to sparse matrices*************************************************
1634 // No special implementation for the Schur product assignment to sparse matrices.
1635 //**********************************************************************************************
1636
1637 //**Multiplication assignment to dense matrices*************************************************
1638 // No special implementation for the multiplication assignment to dense matrices.
1639 //**********************************************************************************************
1640
1641 //**Multiplication assignment to sparse matrices************************************************
1642 // No special implementation for the multiplication assignment to sparse matrices.
1643 //**********************************************************************************************
1644
1645 //**SMP assignment to dense matrices************************************************************
1661 template< typename MT // Type of the target dense matrix
1662 , bool SO > // Storage order of the target dense matrix
1663 friend inline auto smpAssign( DenseMatrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
1664 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1665 {
1667
1668 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1669 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1670
1671 LT A( rhs.lhs_ ); // Evaluation of the right-hand side sparse matrix operand
1672 RT B( rhs.rhs_ ); // Evaluation of the left-hand side dense matrix operand
1673
1674 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1675 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1676 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1677 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1678 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1679 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
1680
1681 smpAssign( *lhs, A * B );
1682 }
1684 //**********************************************************************************************
1685
1686 //**SMP assignment to sparse matrices***********************************************************
1702 template< typename MT // Type of the target sparse matrix
1703 , bool SO > // Storage order of the target sparse matrix
1704 friend inline auto smpAssign( SparseMatrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
1705 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1706 {
1708
1709 using TmpType = If_t< SO, ResultType, OppositeType >;
1710
1717
1718 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1719 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1720
1721 const ForwardFunctor fwd;
1722
1723 const TmpType tmp( rhs );
1724 smpAssign( *lhs, fwd( tmp ) );
1725 }
1727 //**********************************************************************************************
1728
1729 //**Restructuring SMP assignment****************************************************************
1744 template< typename MT // Type of the target matrix
1745 , bool SO > // Storage order of the target matrix
1746 friend inline auto smpAssign( Matrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
1747 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1748 {
1750
1752
1753 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1754 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1755
1756 const ForwardFunctor fwd;
1757
1758 decltype(auto) A( transIf< IsSymmetric_v<MT1> >( rhs.lhs_ ) );
1759 decltype(auto) B( transIf< IsSymmetric_v<MT2> >( rhs.rhs_ ) );
1760
1761 smpAssign( *lhs, fwd( A * B ) );
1762 }
1764 //**********************************************************************************************
1765
1766 //**SMP addition assignment to dense matrices***************************************************
1782 template< typename MT // Type of the target dense matrix
1783 , bool SO > // Storage order of the target dense matrix
1784 friend inline auto smpAddAssign( DenseMatrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
1785 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1786 {
1788
1789 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1790 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1791
1792 LT A( rhs.lhs_ ); // Evaluation of the right-hand side sparse matrix operand
1793 RT B( rhs.rhs_ ); // Evaluation of the left-hand side dense matrix operand
1794
1795 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1796 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1797 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1798 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1799 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1800 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
1801
1802 smpAddAssign( *lhs, A * B );
1803 }
1805 //**********************************************************************************************
1806
1807 //**Restructuring SMP addition assignment*******************************************************
1822 template< typename MT // Type of the target matrix
1823 , bool SO > // Storage order of the target matrix
1824 friend inline auto smpAddAssign( Matrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
1825 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1826 {
1828
1830
1831 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1832 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1833
1834 const ForwardFunctor fwd;
1835
1836 decltype(auto) A( transIf< IsSymmetric_v<MT1> >( rhs.lhs_ ) );
1837 decltype(auto) B( transIf< IsSymmetric_v<MT2> >( rhs.rhs_ ) );
1838
1839 smpAddAssign( *lhs, fwd( A * B ) );
1840 }
1842 //**********************************************************************************************
1843
1844 //**SMP addition assignment to sparse matrices**************************************************
1845 // No special implementation for the SMP addition assignment to sparse matrices.
1846 //**********************************************************************************************
1847
1848 //**SMP subtraction assignment to dense matrices************************************************
1864 template< typename MT // Type of the target dense matrix
1865 , bool SO > // Storage order of the target dense matrix
1866 friend inline auto smpSubAssign( DenseMatrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
1867 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1868 {
1870
1871 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1872 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1873
1874 LT A( rhs.lhs_ ); // Evaluation of the right-hand side sparse matrix operand
1875 RT B( rhs.rhs_ ); // Evaluation of the left-hand side dense matrix operand
1876
1877 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1878 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1879 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1880 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1881 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1882 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
1883
1884 smpSubAssign( *lhs, A * B );
1885 }
1887 //**********************************************************************************************
1888
1889 //**Restructuring SMP subtraction assignment****************************************************
1904 template< typename MT // Type of the target matrix
1905 , bool SO > // Storage order of the target matrix
1906 friend inline auto smpSubAssign( Matrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
1907 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1908 {
1910
1912
1913 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1914 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1915
1916 const ForwardFunctor fwd;
1917
1918 decltype(auto) A( transIf< IsSymmetric_v<MT1> >( rhs.lhs_ ) );
1919 decltype(auto) B( transIf< IsSymmetric_v<MT2> >( rhs.rhs_ ) );
1920
1921 smpSubAssign( *lhs, fwd( A * B ) );
1922 }
1924 //**********************************************************************************************
1925
1926 //**SMP subtraction assignment to sparse matrices***********************************************
1927 // No special implementation for the SMP subtraction assignment to sparse matrices.
1928 //**********************************************************************************************
1929
1930 //**SMP Schur product assignment to dense matrices**********************************************
1943 template< typename MT // Type of the target dense matrix
1944 , bool SO > // Storage order of the target dense matrix
1945 friend inline void smpSchurAssign( DenseMatrix<MT,SO>& lhs, const TSMatTDMatMultExpr& rhs )
1946 {
1948
1952
1953 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1954 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1955
1956 const ResultType tmp( rhs );
1957 smpSchurAssign( *lhs, tmp );
1958 }
1960 //**********************************************************************************************
1961
1962 //**SMP Schur product assignment to sparse matrices*********************************************
1963 // No special implementation for the SMP Schur product assignment to sparse matrices.
1964 //**********************************************************************************************
1965
1966 //**SMP multiplication assignment to dense matrices*********************************************
1967 // No special implementation for the SMP multiplication assignment to dense matrices.
1968 //**********************************************************************************************
1969
1970 //**SMP multiplication assignment to sparse matrices********************************************
1971 // No special implementation for the SMP multiplication assignment to sparse matrices.
1972 //**********************************************************************************************
1973
1974 //**Compile time checks*************************************************************************
1983 //**********************************************************************************************
1984};
1985//*************************************************************************************************
1986
1987
1988
1989
1990//=================================================================================================
1991//
1992// GLOBAL BINARY ARITHMETIC OPERATORS
1993//
1994//=================================================================================================
1995
1996//*************************************************************************************************
2009template< typename MT1 // Type of the left-hand side dense matrix
2010 , typename MT2 // Type of the right-hand side sparse matrix
2011 , DisableIf_t< ( IsIdentity_v<MT1> &&
2012 IsSame_v< ElementType_t<MT1>, ElementType_t<MT2> > ) ||
2013 IsZero_v<MT1> >* = nullptr >
2014inline const TSMatTDMatMultExpr<MT1,MT2,false,false,false,false>
2015 tsmattdmatmult( const SparseMatrix<MT1,true>& lhs, const DenseMatrix<MT2,true>& rhs )
2016{
2018
2019 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == (*rhs).rows(), "Invalid matrix sizes" );
2020
2021 return TSMatTDMatMultExpr<MT1,MT2,false,false,false,false>( *lhs, *rhs );
2022}
2024//*************************************************************************************************
2025
2026
2027//*************************************************************************************************
2041template< typename MT1 // Type of the left-hand side sparse matrix
2042 , typename MT2 // Type of the right-hand side dense matrix
2043 , EnableIf_t< IsIdentity_v<MT1> &&
2044 IsSame_v< ElementType_t<MT1>, ElementType_t<MT2> > >* = nullptr >
2045inline const MT2&
2046 tsmattdmatmult( const SparseMatrix<MT1,true>& lhs, const DenseMatrix<MT2,true>& rhs )
2047{
2049
2050 MAYBE_UNUSED( lhs );
2051
2052 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == (*rhs).rows(), "Invalid matrix sizes" );
2053
2054 return (*rhs);
2055}
2057//*************************************************************************************************
2058
2059
2060//*************************************************************************************************
2073template< typename MT1 // Type of the left-hand side dense matrix
2074 , typename MT2 // Type of the right-hand side sparse matrix
2075 , EnableIf_t< IsZero_v<MT1> >* = nullptr >
2076inline decltype(auto)
2077 tsmattdmatmult( const SparseMatrix<MT1,true>& lhs, const DenseMatrix<MT2,true>& rhs )
2078{
2080
2081 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == (*rhs).rows(), "Invalid matrix sizes" );
2082
2083 using ReturnType = const MultTrait_t< ResultType_t<MT1>, ResultType_t<MT2> >;
2084
2087
2088 return ReturnType( (*lhs).rows(), (*rhs).columns() );
2089}
2091//*************************************************************************************************
2092
2093
2094//*************************************************************************************************
2123template< typename MT1 // Type of the left-hand side sparse matrix
2124 , typename MT2 > // Type of the right-hand side dense matrix
2125inline decltype(auto)
2126 operator*( const SparseMatrix<MT1,true>& lhs, const DenseMatrix<MT2,true>& rhs )
2127{
2129
2130 if( (*lhs).columns() != (*rhs).rows() ) {
2131 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
2132 }
2133
2134 return tsmattdmatmult( *lhs, *rhs );
2135}
2136//*************************************************************************************************
2137
2138
2139
2140
2141//=================================================================================================
2142//
2143// GLOBAL FUNCTIONS
2144//
2145//=================================================================================================
2146
2147//*************************************************************************************************
2171template< typename MT1 // Type of the left-hand side dense matrix
2172 , typename MT2 // Type of the right-hand side dense matrix
2173 , bool SF // Symmetry flag
2174 , bool HF // Hermitian flag
2175 , bool LF // Lower flag
2176 , bool UF > // Upper flag
2177inline decltype(auto) declsym( const TSMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2178{
2180
2181 if( !isSquare( dm ) ) {
2182 BLAZE_THROW_INVALID_ARGUMENT( "Invalid symmetric matrix specification" );
2183 }
2184
2185 using ReturnType = const TSMatTDMatMultExpr<MT1,MT2,true,HF,LF,UF>;
2186 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2187}
2189//*************************************************************************************************
2190
2191
2192//*************************************************************************************************
2216template< typename MT1 // Type of the left-hand side dense matrix
2217 , typename MT2 // Type of the right-hand side dense matrix
2218 , bool SF // Symmetry flag
2219 , bool HF // Hermitian flag
2220 , bool LF // Lower flag
2221 , bool UF > // Upper flag
2222inline decltype(auto) declherm( const TSMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2223{
2225
2226 if( !isSquare( dm ) ) {
2227 BLAZE_THROW_INVALID_ARGUMENT( "Invalid Hermitian matrix specification" );
2228 }
2229
2230 using ReturnType = const TSMatTDMatMultExpr<MT1,MT2,SF,true,LF,UF>;
2231 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2232}
2234//*************************************************************************************************
2235
2236
2237//*************************************************************************************************
2261template< typename MT1 // Type of the left-hand side dense matrix
2262 , typename MT2 // Type of the right-hand side dense matrix
2263 , bool SF // Symmetry flag
2264 , bool HF // Hermitian flag
2265 , bool LF // Lower flag
2266 , bool UF > // Upper flag
2267inline decltype(auto) decllow( const TSMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2268{
2270
2271 if( !isSquare( dm ) ) {
2272 BLAZE_THROW_INVALID_ARGUMENT( "Invalid lower matrix specification" );
2273 }
2274
2275 using ReturnType = const TSMatTDMatMultExpr<MT1,MT2,SF,HF,true,UF>;
2276 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2277}
2279//*************************************************************************************************
2280
2281
2282//*************************************************************************************************
2306template< typename MT1 // Type of the left-hand side dense matrix
2307 , typename MT2 // Type of the right-hand side dense matrix
2308 , bool SF // Symmetry flag
2309 , bool HF // Hermitian flag
2310 , bool UF > // Upper flag
2311inline decltype(auto) declunilow( const TSMatTDMatMultExpr<MT1,MT2,SF,HF,false,UF>& dm )
2312{
2314
2315 if( !isSquare( dm ) ) {
2316 BLAZE_THROW_INVALID_ARGUMENT( "Invalid unilower matrix specification" );
2317 }
2318
2319 return declunilow( decllow( *dm ) );
2320}
2322//*************************************************************************************************
2323
2324
2325//*************************************************************************************************
2349template< typename MT1 // Type of the left-hand side dense matrix
2350 , typename MT2 // Type of the right-hand side dense matrix
2351 , bool SF // Symmetry flag
2352 , bool HF // Hermitian flag
2353 , bool UF > // Upper flag
2354inline decltype(auto) declstrlow( const TSMatTDMatMultExpr<MT1,MT2,SF,HF,false,UF>& dm )
2355{
2357
2358 if( !isSquare( dm ) ) {
2359 BLAZE_THROW_INVALID_ARGUMENT( "Invalid strictly lower matrix specification" );
2360 }
2361
2362 return declstrlow( decllow( *dm ) );
2363}
2365//*************************************************************************************************
2366
2367
2368//*************************************************************************************************
2392template< typename MT1 // Type of the left-hand side dense matrix
2393 , typename MT2 // Type of the right-hand side dense matrix
2394 , bool SF // Symmetry flag
2395 , bool HF // Hermitian flag
2396 , bool LF // Lower flag
2397 , bool UF > // Upper flag
2398inline decltype(auto) declupp( const TSMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2399{
2401
2402 if( !isSquare( dm ) ) {
2403 BLAZE_THROW_INVALID_ARGUMENT( "Invalid upper matrix specification" );
2404 }
2405
2406 using ReturnType = const TSMatTDMatMultExpr<MT1,MT2,SF,HF,LF,true>;
2407 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2408}
2410//*************************************************************************************************
2411
2412
2413//*************************************************************************************************
2437template< typename MT1 // Type of the left-hand side dense matrix
2438 , typename MT2 // Type of the right-hand side dense matrix
2439 , bool SF // Symmetry flag
2440 , bool HF // Hermitian flag
2441 , bool LF > // Lower flag
2442inline decltype(auto) decluniupp( const TSMatTDMatMultExpr<MT1,MT2,SF,HF,LF,false>& dm )
2443{
2445
2446 if( !isSquare( dm ) ) {
2447 BLAZE_THROW_INVALID_ARGUMENT( "Invalid uniupper matrix specification" );
2448 }
2449
2450 return decluniupp( declupp( *dm ) );
2451}
2453//*************************************************************************************************
2454
2455
2456//*************************************************************************************************
2480template< typename MT1 // Type of the left-hand side dense matrix
2481 , typename MT2 // Type of the right-hand side dense matrix
2482 , bool SF // Symmetry flag
2483 , bool HF // Hermitian flag
2484 , bool LF > // Lower flag
2485inline decltype(auto) declstrupp( const TSMatTDMatMultExpr<MT1,MT2,SF,HF,LF,false>& dm )
2486{
2488
2489 if( !isSquare( dm ) ) {
2490 BLAZE_THROW_INVALID_ARGUMENT( "Invalid strictly upper matrix specification" );
2491 }
2492
2493 return declstrupp( declupp( *dm ) );
2494}
2496//*************************************************************************************************
2497
2498
2499//*************************************************************************************************
2523template< typename MT1 // Type of the left-hand side dense matrix
2524 , typename MT2 // Type of the right-hand side dense matrix
2525 , bool SF // Symmetry flag
2526 , bool HF // Hermitian flag
2527 , bool LF // Lower flag
2528 , bool UF > // Upper flag
2529inline decltype(auto) decldiag( const TSMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2530{
2532
2533 if( !isSquare( dm ) ) {
2534 BLAZE_THROW_INVALID_ARGUMENT( "Invalid diagonal matrix specification" );
2535 }
2536
2537 using ReturnType = const TSMatTDMatMultExpr<MT1,MT2,SF,HF,true,true>;
2538 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2539}
2541//*************************************************************************************************
2542
2543
2544
2545
2546//=================================================================================================
2547//
2548// SIZE SPECIALIZATIONS
2549//
2550//=================================================================================================
2551
2552//*************************************************************************************************
2554template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2555struct Size< TSMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 0UL >
2556 : public Size<MT1,0UL>
2557{};
2558
2559template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2560struct Size< TSMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 1UL >
2561 : public Size<MT2,1UL>
2562{};
2564//*************************************************************************************************
2565
2566
2567
2568
2569//=================================================================================================
2570//
2571// ISALIGNED SPECIALIZATIONS
2572//
2573//=================================================================================================
2574
2575//*************************************************************************************************
2577template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2578struct IsAligned< TSMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2579 : public IsAligned<MT2>
2580{};
2582//*************************************************************************************************
2583
2584} // namespace blaze
2585
2586#endif
Header file for auxiliary alias declarations.
typename T::CompositeType CompositeType_t
Alias declaration for nested CompositeType type definitions.
Definition: Aliases.h:110
typename T::ResultType ResultType_t
Alias declaration for nested ResultType type definitions.
Definition: Aliases.h:450
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.
Definition: Aliases.h:190
typename T::OppositeType OppositeType_t
Alias declaration for nested OppositeType type definitions.
Definition: Aliases.h:310
typename T::TransposeType TransposeType_t
Alias declaration for nested TransposeType type definitions.
Definition: Aliases.h:550
Header file for run time assertion macros.
Header file for the blaze::checked and blaze::unchecked instances.
Constraints on the storage order of matrix types.
Header file for the conjugate shim.
Header file for the decldiag trait.
Header file for the DeclDiag functor.
Header file for the declherm trait.
Header file for the DeclHerm functor.
Header file for the decllow trait.
Header file for the DeclLow functor.
Header file for the declsym trait.
Header file for the DeclSym functor.
Header file for the declupp trait.
Header file for the DeclUpp functor.
Header file for the EnableIf class template.
Header file for the function trace functionality.
Header file for the If class template.
Header file for the IntegralConstant class template.
Header file for the IsAligned type trait.
Header file for the IsBuiltin type trait.
Header file for the IsColumnMajorMatrix type trait.
Header file for the IsComputation type trait class.
Header file for the isDefault shim.
Header file for the IsDiagonal type trait.
Header file for the IsExpression type trait class.
Header file for the IsIdentity type trait.
Header file for the IsLower type trait.
Header file for the IsResizable type trait.
Header file for the IsRowMajorMatrix type trait.
Header file for the IsStrictlyLower type trait.
Header file for the IsStrictlyUpper type trait.
Header file for the IsSymmetric type trait.
Header file for the IsTriangular type trait.
Header file for the IsUpper type trait.
Header file for the MAYBE_UNUSED function template.
Header file for the multiplication trait.
Header file for the Noop functor.
Header file for the prevMultiple shim.
Constraints on the storage order of matrix types.
Constraint on the data type.
Constraint on the data type.
Base class for dense matrices.
Definition: DenseMatrix.h:82
Base class for sparse matrices.
Definition: SparseMatrix.h:77
Expression object for transpose sparse matrix-transpose dense matrix multiplications.
Definition: TSMatTDMatMultExpr.h:131
RightOperand rightOperand() const noexcept
Returns the right-hand side transpose dense matrix operand.
Definition: TSMatTDMatMultExpr.h:391
static constexpr bool evaluateRight
Compilation switch for the composite type of the right-hand side dense matrix expression.
Definition: TSMatTDMatMultExpr.h:149
ElementType_t< RT1 > ET1
Element type of the left-hand side dense matrix expression.
Definition: TSMatTDMatMultExpr.h:136
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: TSMatTDMatMultExpr.h:425
If_t< IsExpression_v< MT2 >, const MT2, const MT2 & > RightOperand
Composite type of the right-hand side dense matrix expression.
Definition: TSMatTDMatMultExpr.h:257
static constexpr bool SYM
Flag for symmetric matrices.
Definition: TSMatTDMatMultExpr.h:153
static constexpr bool simdEnabled
Compilation switch for the expression template evaluation strategy.
Definition: TSMatTDMatMultExpr.h:268
OppositeType_t< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: TSMatTDMatMultExpr.h:247
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: TSMatTDMatMultExpr.h:361
CompositeType_t< MT1 > CT1
Composite type of the left-hand side sparse matrix expression.
Definition: TSMatTDMatMultExpr.h:138
static constexpr bool LOW
Flag for lower matrices.
Definition: TSMatTDMatMultExpr.h:155
ElementType_t< ResultType > ElementType
Resulting element type.
Definition: TSMatTDMatMultExpr.h:249
static constexpr bool evaluateLeft
Compilation switch for the composite type of the left-hand side sparse matrix expression.
Definition: TSMatTDMatMultExpr.h:144
CompositeType_t< MT2 > CT2
Composite type of the right-hand side dense matrix expression.
Definition: TSMatTDMatMultExpr.h:139
ResultType_t< MT1 > RT1
Result type of the left-hand side sparse matrix expression.
Definition: TSMatTDMatMultExpr.h:134
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: TSMatTDMatMultExpr.h:371
const ResultType CompositeType
Data type for composite expression templates.
Definition: TSMatTDMatMultExpr.h:251
static constexpr bool UPP
Flag for upper matrices.
Definition: TSMatTDMatMultExpr.h:156
typename If_t< HERM, DeclHermTrait< MultTrait_t< RT1, RT2 > >, If_t< SYM, DeclSymTrait< MultTrait_t< RT1, RT2 > >, If_t< LOW, If_t< UPP, DeclDiagTrait< MultTrait_t< RT1, RT2 > >, DeclLowTrait< MultTrait_t< RT1, RT2 > > >, If_t< UPP, DeclUppTrait< MultTrait_t< RT1, RT2 > >, MultTrait< RT1, RT2 > > > > >::Type ResultType
Result type for expression template evaluations.
Definition: TSMatTDMatMultExpr.h:245
TSMatTDMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the TSMatTDMatMultExpr class.
Definition: TSMatTDMatMultExpr.h:281
LeftOperand leftOperand() const noexcept
Returns the left-hand side transpose sparse matrix operand.
Definition: TSMatTDMatMultExpr.h:381
RightOperand rhs_
Right-hand side dense matrix of the multiplication expression.
Definition: TSMatTDMatMultExpr.h:443
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: TSMatTDMatMultExpr.h:415
ResultType_t< MT2 > RT2
Result type of the right-hand side dense matrix expression.
Definition: TSMatTDMatMultExpr.h:135
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: TSMatTDMatMultExpr.h:435
ElementType_t< RT2 > ET2
Element type of the right-hand side sparse matrix expression.
Definition: TSMatTDMatMultExpr.h:137
If_t< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: TSMatTDMatMultExpr.h:263
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: TSMatTDMatMultExpr.h:345
const ElementType ReturnType
Return type for expression template evaluations.
Definition: TSMatTDMatMultExpr.h:250
If_t< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side sparse matrix operand.
Definition: TSMatTDMatMultExpr.h:260
LeftOperand lhs_
Left-hand side sparse matrix of the multiplication expression.
Definition: TSMatTDMatMultExpr.h:442
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: TSMatTDMatMultExpr.h:296
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: TSMatTDMatMultExpr.h:403
TransposeType_t< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: TSMatTDMatMultExpr.h:248
static constexpr bool HERM
Flag for Hermitian matrices.
Definition: TSMatTDMatMultExpr.h:154
If_t< IsExpression_v< MT1 >, const MT1, const MT1 & > LeftOperand
Composite type of the left-hand side sparse matrix expression.
Definition: TSMatTDMatMultExpr.h:254
static constexpr bool smpAssignable
Compilation switch for the expression template assignment strategy.
Definition: TSMatTDMatMultExpr.h:271
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Header file for the Computation base class.
Header file for the DenseMatrix base class.
Header file for the MatMatMultExpr base class.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:137
decltype(auto) transIf(const DenseMatrix< MT, SO > &dm)
Conditional calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:832
decltype(auto) min(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise minimum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1339
decltype(auto) max(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise maximum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1375
decltype(auto) declstrupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as strictly upper.
Definition: DMatDeclStrUppExpr.h:1003
decltype(auto) conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition: DMatMapExpr.h:1464
decltype(auto) decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as diagonal.
Definition: DMatDeclDiagExpr.h:978
decltype(auto) declstrlow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as strictly lower.
Definition: DMatDeclStrLowExpr.h:1003
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:812
decltype(auto) declupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as upper.
Definition: DMatDeclUppExpr.h:1004
decltype(auto) decllow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as lower.
Definition: DMatDeclLowExpr.h:1004
decltype(auto) decluniupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as uniupper.
Definition: DMatDeclUniUppExpr.h:1005
decltype(auto) declherm(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as Hermitian.
Definition: DMatDeclHermExpr.h:1005
decltype(auto) declsym(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as symmetric.
Definition: DMatDeclSymExpr.h:1005
decltype(auto) declunilow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as unilower.
Definition: DMatDeclUniLowExpr.h:1004
bool isDefault(const DiagonalMatrix< MT, SO, DF > &m)
Returns whether the given diagonal matrix is in default state.
Definition: DiagonalMatrix.h:169
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.
Definition: Symmetric.h:79
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition: RowMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.
Definition: StorageOrder.h:84
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.
Definition: RequiresEvaluation.h:81
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.
Definition: MatMatMultExpr.h:103
#define BLAZE_CONSTRAINT_MUST_NOT_BE_ZERO_TYPE(T)
Constraint on the data type.
Definition: Zero.h:81
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.
Definition: DenseMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.
Definition: SparseMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition: ColumnMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_ZERO_TYPE(T)
Constraint on the data type.
Definition: Zero.h:61
BLAZE_ALWAYS_INLINE constexpr auto prevMultiple(T1 value, T2 factor) noexcept
Rounds down an integral value to the previous multiple of a given factor.
Definition: PrevMultiple.h:68
constexpr void reset(Matrix< MT, SO > &matrix)
Resetting the given matrix.
Definition: Matrix.h:806
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:584
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:518
constexpr size_t size(const Matrix< MT, SO > &matrix) noexcept
Returns the total number of elements of the matrix.
Definition: Matrix.h:676
bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition: Matrix.h:1383
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:137
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.
Definition: Assert.h:101
auto smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:162
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
auto smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:194
auto smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:131
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:158
constexpr void MAYBE_UNUSED(const Args &...)
Suppression of unused parameter warnings.
Definition: MaybeUnused.h:81
typename If< Condition >::template Type< T1, T2 > If_t
Auxiliary alias template for the If class template.
Definition: If.h:108
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.
Definition: Exception.h:331
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.
Definition: Exception.h:235
typename EnableIf<!Condition, T >::Type DisableIf_t
Auxiliary type for the EnableIf class template.
Definition: EnableIf.h:175
#define BLAZE_FUNCTION_TRACE
Function trace macro.
Definition: FunctionTrace.h:94
constexpr Unchecked unchecked
Global Unchecked instance.
Definition: Check.h:146
Header file for the exception macros of the math module.
Constraints on the storage order of matrix types.
Header file for all forward declarations for expression class templates.
Header file for the Size type trait.
Header file for the reset shim.
Header file for the serial shim.
Base class for all compute expression templates.
Definition: Computation.h:68
Base template for the DeclDiagTrait class.
Definition: DeclDiagTrait.h:127
Generic wrapper for the decldiag() function.
Definition: DeclDiag.h:61
Base template for the DeclHermTrait class.
Definition: DeclHermTrait.h:126
Generic wrapper for the declherm() function.
Definition: DeclHerm.h:61
Base template for the DeclLowTrait class.
Definition: DeclLowTrait.h:126
Generic wrapper for the decllow() function.
Definition: DeclLow.h:61
Base template for the DeclSymTrait class.
Definition: DeclSymTrait.h:126
Generic wrapper for the declsym() function.
Definition: DeclSym.h:61
Base template for the DeclUppTrait class.
Definition: DeclUppTrait.h:126
Generic wrapper for the declupp() function.
Definition: DeclUpp.h:61
Base class for all matrix/matrix multiplication expression templates.
Definition: MatMatMultExpr.h:71
Base template for the MultTrait class.
Definition: MultTrait.h:130
Generic wrapper for the null function.
Definition: Noop.h:62
System settings for performance optimizations.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
Header file for the IsZero type trait.
Header file for the RequiresEvaluation type trait.
Header file for basic type definitions.
Header file for the generic max algorithm.
Header file for the generic min algorithm.