Blaze 3.9
SMatTDMatMultExpr.h
Go to the documentation of this file.
1//=================================================================================================
33//=================================================================================================
34
35#ifndef _BLAZE_MATH_EXPRESSIONS_SMATTDMATMULTEXPR_H_
36#define _BLAZE_MATH_EXPRESSIONS_SMATTDMATMULTEXPR_H_
37
38
39//*************************************************************************************************
40// Includes
41//*************************************************************************************************
42
43#include <blaze/math/Aliases.h>
93#include <blaze/util/Assert.h>
94#include <blaze/util/EnableIf.h>
98#include <blaze/util/mpl/If.h>
99#include <blaze/util/Types.h>
101
102
103namespace blaze {
104
105//=================================================================================================
106//
107// CLASS SMATTDMATMULTEXPR
108//
109//=================================================================================================
110
111//*************************************************************************************************
118template< typename MT1 // Type of the left-hand side sparse matrix
119 , typename MT2 // Type of the right-hand side dense matrix
120 , bool SF // Symmetry flag
121 , bool HF // Hermitian flag
122 , bool LF // Lower flag
123 , bool UF > // Upper flag
125 : public MatMatMultExpr< DenseMatrix< SMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, true > >
126 , private Computation
127{
128 private:
129 //**Type definitions****************************************************************************
136 //**********************************************************************************************
137
138 //**********************************************************************************************
140 static constexpr bool evaluateLeft = ( IsComputation_v<MT1> || RequiresEvaluation_v<MT1> );
141 //**********************************************************************************************
142
143 //**********************************************************************************************
145 static constexpr bool evaluateRight = ( IsComputation_v<MT2> || RequiresEvaluation_v<MT2> );
146 //**********************************************************************************************
147
148 //**********************************************************************************************
149 static constexpr bool SYM = ( SF && !( HF || LF || UF ) );
150 static constexpr bool HERM = ( HF && !( LF || UF ) );
151 static constexpr bool LOW = ( LF || ( ( SF || HF ) && UF ) );
152 static constexpr bool UPP = ( UF || ( ( SF || HF ) && LF ) );
153 //**********************************************************************************************
154
155 //**********************************************************************************************
157
162 template< typename T1, typename T2, typename T3 >
163 static constexpr bool CanExploitSymmetry_v = IsSymmetric_v<T3>;
165 //**********************************************************************************************
166
167 //**********************************************************************************************
169
173 template< typename T1, typename T2, typename T3 >
174 static constexpr bool IsEvaluationRequired_v =
175 ( ( evaluateLeft || evaluateRight ) && CanExploitSymmetry_v<T1,T2,T3> );
177 //**********************************************************************************************
178
179 //**********************************************************************************************
181
184 template< typename T1, typename T2, typename T3 >
185 static constexpr bool UseOptimizedKernel_v =
186 ( useOptimizedKernels &&
187 !IsDiagonal_v<T3> &&
188 !IsResizable_v< ElementType_t<T1> > &&
189 !IsResizable_v<ET1> );
191 //**********************************************************************************************
192
193 //**********************************************************************************************
195
198 using ForwardFunctor = If_t< HERM
199 , DeclHerm
200 , If_t< SYM
201 , DeclSym
202 , If_t< LOW
203 , If_t< UPP
204 , DeclDiag
205 , DeclLow >
206 , If_t< UPP
207 , DeclUpp
208 , Noop > > > >;
210 //**********************************************************************************************
211
212 public:
213 //**Type definitions****************************************************************************
216
219
221 using ResultType = typename If_t< HERM
223 , If_t< SYM
225 , If_t< LOW
226 , If_t< UPP
229 , If_t< UPP
231 , MultTrait<RT1,RT2> > > > >::Type;
232
236 using ReturnType = const ElementType;
237 using CompositeType = const ResultType;
238
240 using LeftOperand = If_t< IsExpression_v<MT1>, const MT1, const MT1& >;
241
243 using RightOperand = If_t< IsExpression_v<MT2>, const MT2, const MT2& >;
244
247
250 //**********************************************************************************************
251
252 //**Compilation flags***************************************************************************
254 static constexpr bool simdEnabled = false;
255
257 static constexpr bool smpAssignable =
258 ( !evaluateLeft && MT1::smpAssignable && !evaluateRight && MT2::smpAssignable );
259 //**********************************************************************************************
260
261 //**Constructor*********************************************************************************
267 inline SMatTDMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
268 : lhs_( lhs ) // Left-hand side sparse matrix of the multiplication expression
269 , rhs_( rhs ) // Right-hand side dense matrix of the multiplication expression
270 {
271 BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
272 }
273 //**********************************************************************************************
274
275 //**Access operator*****************************************************************************
282 inline ReturnType operator()( size_t i, size_t j ) const {
283 BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
284 BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
285
286 if( IsDiagonal_v<MT1> ) {
287 return lhs_(i,i) * rhs_(i,j);
288 }
289 else if( IsDiagonal_v<MT2> ) {
290 return lhs_(i,j) * rhs_(j,j);
291 }
292 else if( IsTriangular_v<MT1> || IsTriangular_v<MT2> ) {
293 const size_t begin( ( IsUpper_v<MT1> )
294 ?( ( IsLower_v<MT2> )
295 ?( max( ( IsStrictlyUpper_v<MT1> ? i+1UL : i )
296 , ( IsStrictlyLower_v<MT2> ? j+1UL : j ) ) )
297 :( IsStrictlyUpper_v<MT1> ? i+1UL : i ) )
298 :( ( IsLower_v<MT2> )
299 ?( IsStrictlyLower_v<MT2> ? j+1UL : j )
300 :( 0UL ) ) );
301 const size_t end( ( IsLower_v<MT1> )
302 ?( ( IsUpper_v<MT2> )
303 ?( min( ( IsStrictlyLower_v<MT1> ? i : i+1UL )
304 , ( IsStrictlyUpper_v<MT2> ? j : j+1UL ) ) )
305 :( IsStrictlyLower_v<MT1> ? i : i+1UL ) )
306 :( ( IsUpper_v<MT2> )
307 ?( IsStrictlyUpper_v<MT2> ? j : j+1UL )
308 :( lhs_.columns() ) ) );
309
310 if( begin >= end ) return ElementType();
311
312 const size_t n( end - begin );
313
314 return subvector( row( lhs_, i, unchecked ), begin, n, unchecked ) *
316 }
317 else {
318 return row( lhs_, i, unchecked ) * column( rhs_, j, unchecked );
319 }
320 }
321 //**********************************************************************************************
322
323 //**At function*********************************************************************************
331 inline ReturnType at( size_t i, size_t j ) const {
332 if( i >= lhs_.rows() ) {
333 BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
334 }
335 if( j >= rhs_.columns() ) {
336 BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
337 }
338 return (*this)(i,j);
339 }
340 //**********************************************************************************************
341
342 //**Rows function*******************************************************************************
347 inline size_t rows() const noexcept {
348 return lhs_.rows();
349 }
350 //**********************************************************************************************
351
352 //**Columns function****************************************************************************
357 inline size_t columns() const noexcept {
358 return rhs_.columns();
359 }
360 //**********************************************************************************************
361
362 //**Left operand access*************************************************************************
367 inline LeftOperand leftOperand() const noexcept {
368 return lhs_;
369 }
370 //**********************************************************************************************
371
372 //**Right operand access************************************************************************
377 inline RightOperand rightOperand() const noexcept {
378 return rhs_;
379 }
380 //**********************************************************************************************
381
382 //**********************************************************************************************
388 template< typename T >
389 inline bool canAlias( const T* alias ) const noexcept {
390 return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
391 }
392 //**********************************************************************************************
393
394 //**********************************************************************************************
400 template< typename T >
401 inline bool isAliased( const T* alias ) const noexcept {
402 return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
403 }
404 //**********************************************************************************************
405
406 //**********************************************************************************************
411 inline bool isAligned() const noexcept {
412 return rhs_.isAligned();
413 }
414 //**********************************************************************************************
415
416 //**********************************************************************************************
421 inline bool canSMPAssign() const noexcept {
422 return ( rows() * columns() >= SMP_SMATTDMATMULT_THRESHOLD ) && !IsDiagonal_v<MT2>;
423 }
424 //**********************************************************************************************
425
426 private:
427 //**Member variables****************************************************************************
430 //**********************************************************************************************
431
432 //**Assignment to dense matrices****************************************************************
445 template< typename MT // Type of the target dense matrix
446 , bool SO > // Storage order of the target dense matrix
447 friend inline auto assign( DenseMatrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
449 {
451
452 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
453 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
454
455 LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
456 RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
457
458 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
459 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
460 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
461 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
462 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
463 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
464
465 SMatTDMatMultExpr::selectAssignKernel( *lhs, A, B );
466 }
468 //**********************************************************************************************
469
470 //**Default assignment to dense matrices********************************************************
484 template< typename MT3 // Type of the left-hand side target matrix
485 , typename MT4 // Type of the left-hand side matrix operand
486 , typename MT5 > // Type of the right-hand side matrix operand
487 static inline auto selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
489 {
490 const size_t M( A.rows() );
491 const size_t N( B.columns() );
492
493 BLAZE_INTERNAL_ASSERT( !( SYM || HERM || LOW || UPP ) || M == N, "Broken invariant detected" );
494
495 {
496 size_t j( 0UL );
497
498 for( ; (j+4UL) <= N; j+=4UL ) {
499 for( size_t i=( SYM || HERM || LOW ? j : 0UL ); i<( UPP ? j+4UL : M ); ++i )
500 {
501 const auto end( ( IsUpper_v<MT5> )
502 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j+4UL) : A.upperBound(i,j+4UL) )
503 :( A.end(i) ) );
504 auto element( ( IsLower_v<MT5> )
505 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
506 :( A.begin(i) ) );
507
508 if( element == end ) {
509 reset( C(i,j ) );
510 reset( C(i,j+1UL) );
511 reset( C(i,j+2UL) );
512 reset( C(i,j+3UL) );
513 continue;
514 }
515
516 C(i,j ) = element->value() * B(element->index(),j );
517 C(i,j+1UL) = element->value() * B(element->index(),j+1UL);
518 C(i,j+2UL) = element->value() * B(element->index(),j+2UL);
519 C(i,j+3UL) = element->value() * B(element->index(),j+3UL);
520 ++element;
521 for( ; element!=end; ++element ) {
522 C(i,j ) += element->value() * B(element->index(),j );
523 C(i,j+1UL) += element->value() * B(element->index(),j+1UL);
524 C(i,j+2UL) += element->value() * B(element->index(),j+2UL);
525 C(i,j+3UL) += element->value() * B(element->index(),j+3UL);
526 }
527 }
528 }
529
530 for( ; (j+2UL) <= N; j+=2UL ) {
531 for( size_t i=( SYM || HERM || LOW ? j : 0UL ); i<( UPP ? j+2UL : M ); ++i )
532 {
533 const auto end( ( IsUpper_v<MT5> )
534 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j+2UL) : A.upperBound(i,j+2UL) )
535 :( A.end(i) ) );
536 auto element( ( IsLower_v<MT5> )
537 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
538 :( A.begin(i) ) );
539
540 if( element == end ) {
541 reset( C(i,j ) );
542 reset( C(i,j+1UL) );
543 continue;
544 }
545
546 C(i,j ) = element->value() * B(element->index(),j );
547 C(i,j+1UL) = element->value() * B(element->index(),j+1UL);
548 ++element;
549 for( ; element!=end; ++element ) {
550 C(i,j ) += element->value() * B(element->index(),j );
551 C(i,j+1UL) += element->value() * B(element->index(),j+1UL);
552 }
553 }
554 }
555
556 for( ; j<N; ++j ) {
557 for( size_t i=( SYM || HERM || LOW ? j : 0UL ); i<( UPP ? j+1UL : M ); ++i )
558 {
559 const auto end( ( IsUpper_v<MT5> )
560 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j) : A.upperBound(i,j) )
561 :( A.end(i) ) );
562 auto element( ( IsLower_v<MT5> )
563 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
564 :( A.begin(i) ) );
565
566 if( element == end ) {
567 reset( C(i,j) );
568 continue;
569 }
570
571 C(i,j) = element->value() * B(element->index(),j);
572 ++element;
573 for( ; element!=end; ++element ) {
574 C(i,j) += element->value() * B(element->index(),j);
575 }
576 }
577 }
578 }
579
580 if( SYM || HERM ) {
581 for( size_t j=1UL; j<N; ++j ) {
582 for( size_t i=0UL; i<j; ++i ) {
583 C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
584 }
585 }
586 }
587 else if( LOW && !UPP ) {
588 for( size_t j=1UL; j<N; ++j ) {
589 for( size_t i=0UL; i<j; ++i ) {
590 reset( C(i,j) );
591 }
592 }
593 }
594 else if( !LOW && UPP ) {
595 for( size_t i=1UL; i<M; ++i ) {
596 for( size_t j=0UL; j<i; ++j ) {
597 reset( C(i,j) );
598 }
599 }
600 }
601 }
603 //**********************************************************************************************
604
605 //**Optimized assignment to dense matrices******************************************************
619 template< typename MT3 // Type of the left-hand side target matrix
620 , typename MT4 // Type of the left-hand side matrix operand
621 , typename MT5 > // Type of the right-hand side matrix operand
622 static inline auto selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
623 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
624 {
625 const size_t M( A.rows() );
626 const size_t N( B.columns() );
627
628 BLAZE_INTERNAL_ASSERT( !( SYM || HERM || LOW || UPP ) || M == N, "Broken invariant detected" );
629
630 reset( C );
631
632 {
633 size_t j( 0UL );
634
635 for( ; (j+4UL) <= N; j+=4UL ) {
636 for( size_t i=( SYM || HERM || LOW ? j : 0UL ); i<( UPP ? j+4UL : M ); ++i )
637 {
638 const auto end( ( IsUpper_v<MT5> )
639 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j+4UL) : A.upperBound(i,j+4UL) )
640 :( A.end(i) ) );
641 auto element( ( IsLower_v<MT5> )
642 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
643 :( A.begin(i) ) );
644
645 const size_t nonzeros( end - element );
646 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
647 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
648
649 for( size_t k=0UL; k<kpos; k+=4UL )
650 {
651 const size_t i1( element->index() );
652 const ET1 v1( element->value() );
653 ++element;
654 const size_t i2( element->index() );
655 const ET1 v2( element->value() );
656 ++element;
657 const size_t i3( element->index() );
658 const ET1 v3( element->value() );
659 ++element;
660 const size_t i4( element->index() );
661 const ET1 v4( element->value() );
662 ++element;
663
664 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
665
666 C(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
667 C(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
668 C(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
669 C(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
670 }
671
672 for( ; element!=end; ++element )
673 {
674 const size_t i1( element->index() );
675 const ET1 v1( element->value() );
676
677 C(i,j ) += v1 * B(i1,j );
678 C(i,j+1UL) += v1 * B(i1,j+1UL);
679 C(i,j+2UL) += v1 * B(i1,j+2UL);
680 C(i,j+3UL) += v1 * B(i1,j+3UL);
681 }
682 }
683 }
684
685 for( ; (j+2UL) <= N; j+=2UL ) {
686 for( size_t i=( SYM || HERM || LOW ? j : 0UL ); i<( UPP ? j+2UL : M ); ++i )
687 {
688 const auto end( ( IsUpper_v<MT5> )
689 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j+2UL) : A.upperBound(i,j+2UL) )
690 :( A.end(i) ) );
691 auto element( ( IsLower_v<MT5> )
692 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
693 :( A.begin(i) ) );
694
695 const size_t nonzeros( end - element );
696 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
697 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
698
699 for( size_t k=0UL; k<kpos; k+=4UL )
700 {
701 const size_t i1( element->index() );
702 const ET1 v1( element->value() );
703 ++element;
704 const size_t i2( element->index() );
705 const ET1 v2( element->value() );
706 ++element;
707 const size_t i3( element->index() );
708 const ET1 v3( element->value() );
709 ++element;
710 const size_t i4( element->index() );
711 const ET1 v4( element->value() );
712 ++element;
713
714 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
715
716 C(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
717 C(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
718 }
719
720 for( ; element!=end; ++element )
721 {
722 const size_t i1( element->index() );
723 const ET1 v1( element->value() );
724
725 C(i,j ) += v1 * B(i1,j );
726 C(i,j+1UL) += v1 * B(i1,j+1UL);
727 }
728 }
729 }
730
731 for( ; j<N; ++j ) {
732 for( size_t i=( SYM || HERM || LOW ? j : 0UL ); i<( UPP ? j+1UL : M ); ++i )
733 {
734 const auto end( ( IsUpper_v<MT5> )
735 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j) : A.upperBound(i,j) )
736 :( A.end(i) ) );
737 auto element( ( IsLower_v<MT5> )
738 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
739 :( A.begin(i) ) );
740
741 const size_t nonzeros( end - element );
742 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
743 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
744
745 for( size_t k=0UL; k<kpos; k+=4UL )
746 {
747 const size_t i1( element->index() );
748 const ET1 v1( element->value() );
749 ++element;
750 const size_t i2( element->index() );
751 const ET1 v2( element->value() );
752 ++element;
753 const size_t i3( element->index() );
754 const ET1 v3( element->value() );
755 ++element;
756 const size_t i4( element->index() );
757 const ET1 v4( element->value() );
758 ++element;
759
760 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
761
762 C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
763 }
764
765 for( ; element!=end; ++element )
766 {
767 const size_t i1( element->index() );
768 const ET1 v1( element->value() );
769
770 C(i,j) += v1 * B(i1,j);
771 }
772 }
773 }
774 }
775
776 if( SYM || HERM ) {
777 for( size_t j=1UL; j<N; ++j ) {
778 for( size_t i=0UL; i<j; ++i ) {
779 C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
780 }
781 }
782 }
783 }
785 //**********************************************************************************************
786
787 //**Assignment to sparse matrices***************************************************************
800 template< typename MT // Type of the target sparse matrix
801 , bool SO > // Storage order of the target sparse matrix
802 friend inline auto assign( SparseMatrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
803 -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
804 {
806
807 using TmpType = If_t< SO, ResultType, OppositeType >;
808
815
816 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
817 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
818
819 const ForwardFunctor fwd;
820
821 const TmpType tmp( serial( rhs ) );
822 assign( *lhs, fwd( tmp ) );
823 }
825 //**********************************************************************************************
826
827 //**Restructuring assignment********************************************************************
842 template< typename MT // Type of the target matrix
843 , bool SO > // Storage order of the target matrix
844 friend inline auto assign( Matrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
845 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
846 {
848
849 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
850 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
851
852 const ForwardFunctor fwd;
853
854 assign( *lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
855 }
857 //**********************************************************************************************
858
859 //**Addition assignment to dense matrices*******************************************************
872 template< typename MT // Type of the target dense matrix
873 , bool SO > // Storage order of the target dense matrix
874 friend inline auto addAssign( DenseMatrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
875 -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
876 {
878
879 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
880 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
881
882 LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
883 RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
884
885 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
886 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
887 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
888 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
889 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
890 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
891
892 SMatTDMatMultExpr::selectAddAssignKernel( *lhs, A, B );
893 }
895 //**********************************************************************************************
896
897 //**Default addition assignment to dense matrices***********************************************
911 template< typename MT3 // Type of the left-hand side target matrix
912 , typename MT4 // Type of the left-hand side matrix operand
913 , typename MT5 > // Type of the right-hand side matrix operand
914 static inline auto selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
915 -> DisableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
916 {
917 const size_t M( A.rows() );
918 const size_t N( B.columns() );
919
920 BLAZE_INTERNAL_ASSERT( !( LOW || UPP ) || M == N, "Broken invariant detected" );
921
922 {
923 size_t j( 0UL );
924
925 for( ; (j+4UL) <= N; j+=4UL ) {
926 for( size_t i=( LOW ? j : 0UL ); i<( UPP ? j+4UL : M ); ++i )
927 {
928 const auto end( ( IsUpper_v<MT5> )
929 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j+4UL) : A.upperBound(i,j+4UL) )
930 :( A.end(i) ) );
931 auto element( ( IsLower_v<MT5> )
932 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
933 :( A.begin(i) ) );
934
935 for( ; element!=end; ++element ) {
936 C(i,j ) += element->value() * B(element->index(),j );
937 C(i,j+1UL) += element->value() * B(element->index(),j+1UL);
938 C(i,j+2UL) += element->value() * B(element->index(),j+2UL);
939 C(i,j+3UL) += element->value() * B(element->index(),j+3UL);
940 }
941 }
942 }
943
944 for( ; (j+2UL) <= N; j+=2UL ) {
945 for( size_t i=( LOW ? j : 0UL ); i<( UPP ? j+2UL : M ); ++i )
946 {
947 const auto end( ( IsUpper_v<MT5> )
948 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j+2UL) : A.upperBound(i,j+2UL) )
949 :( A.end(i) ) );
950 auto element( ( IsLower_v<MT5> )
951 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
952 :( A.begin(i) ) );
953
954 for( ; element!=end; ++element ) {
955 C(i,j ) += element->value() * B(element->index(),j );
956 C(i,j+1UL) += element->value() * B(element->index(),j+1UL);
957 }
958 }
959 }
960
961 for( ; j<N; ++j ) {
962 for( size_t i=( LOW ? j : 0UL ); i<( UPP ? j+1UL : M ); ++i )
963 {
964 const auto end( ( IsUpper_v<MT5> )
965 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j) : A.upperBound(i,j) )
966 :( A.end(i) ) );
967 auto element( ( IsLower_v<MT5> )
968 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
969 :( A.begin(i) ) );
970
971 for( ; element!=end; ++element ) {
972 C(i,j) += element->value() * B(element->index(),j);
973 }
974 }
975 }
976 }
977 }
979 //**********************************************************************************************
980
981 //**Optimized addition assignment to dense matrices*********************************************
995 template< typename MT3 // Type of the left-hand side target matrix
996 , typename MT4 // Type of the left-hand side matrix operand
997 , typename MT5 > // Type of the right-hand side matrix operand
998 static inline auto selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
999 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1000 {
1001 const size_t M( A.rows() );
1002 const size_t N( B.columns() );
1003
1004 BLAZE_INTERNAL_ASSERT( !( LOW || UPP ) || M == N, "Broken invariant detected" );
1005
1006 {
1007 size_t j( 0UL );
1008
1009 for( ; (j+4UL) <= N; j+=4UL ) {
1010 for( size_t i=( LOW ? j : 0UL ); i<( UPP ? j+4UL : M ); ++i )
1011 {
1012 const auto end( ( IsUpper_v<MT5> )
1013 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j+4UL) : A.upperBound(i,j+4UL) )
1014 :( A.end(i) ) );
1015 auto element( ( IsLower_v<MT5> )
1016 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
1017 :( A.begin(i) ) );
1018
1019 const size_t nonzeros( end - element );
1020 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1021 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1022
1023 for( size_t k=0UL; k<kpos; k+=4UL )
1024 {
1025 const size_t i1( element->index() );
1026 const ET1 v1( element->value() );
1027 ++element;
1028 const size_t i2( element->index() );
1029 const ET1 v2( element->value() );
1030 ++element;
1031 const size_t i3( element->index() );
1032 const ET1 v3( element->value() );
1033 ++element;
1034 const size_t i4( element->index() );
1035 const ET1 v4( element->value() );
1036 ++element;
1037
1038 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1039
1040 C(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1041 C(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1042 C(i,j+2UL) += v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1043 C(i,j+3UL) += v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1044 }
1045
1046 for( ; element!=end; ++element )
1047 {
1048 const size_t i1( element->index() );
1049 const ET1 v1( element->value() );
1050
1051 C(i,j ) += v1 * B(i1,j );
1052 C(i,j+1UL) += v1 * B(i1,j+1UL);
1053 C(i,j+2UL) += v1 * B(i1,j+2UL);
1054 C(i,j+3UL) += v1 * B(i1,j+3UL);
1055 }
1056 }
1057 }
1058
1059 for( ; (j+2UL) <= N; j+=2UL ) {
1060 for( size_t i=( LOW ? j : 0UL ); i<( UPP ? j+2UL : M ); ++i )
1061 {
1062 const auto end( ( IsUpper_v<MT5> )
1063 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j+2UL) : A.upperBound(i,j+2UL) )
1064 :( A.end(i) ) );
1065 auto element( ( IsLower_v<MT5> )
1066 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
1067 :( A.begin(i) ) );
1068
1069 const size_t nonzeros( end - element );
1070 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1071 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1072
1073 for( size_t k=0UL; k<kpos; k+=4UL )
1074 {
1075 const size_t i1( element->index() );
1076 const ET1 v1( element->value() );
1077 ++element;
1078 const size_t i2( element->index() );
1079 const ET1 v2( element->value() );
1080 ++element;
1081 const size_t i3( element->index() );
1082 const ET1 v3( element->value() );
1083 ++element;
1084 const size_t i4( element->index() );
1085 const ET1 v4( element->value() );
1086 ++element;
1087
1088 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1089
1090 C(i,j ) += v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1091 C(i,j+1UL) += v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1092 }
1093
1094 for( ; element!=end; ++element )
1095 {
1096 const size_t i1( element->index() );
1097 const ET1 v1( element->value() );
1098
1099 C(i,j ) += v1 * B(i1,j );
1100 C(i,j+1UL) += v1 * B(i1,j+1UL);
1101 }
1102 }
1103 }
1104
1105 for( ; j<N; ++j ) {
1106 for( size_t i=( LOW ? j : 0UL ); i<( UPP ? j+1UL : M ); ++i )
1107 {
1108 const auto end( ( IsUpper_v<MT5> )
1109 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j) : A.upperBound(i,j) )
1110 :( A.end(i) ) );
1111 auto element( ( IsLower_v<MT5> )
1112 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
1113 :( A.begin(i) ) );
1114
1115 const size_t nonzeros( end - element );
1116 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1117 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1118
1119 for( size_t k=0UL; k<kpos; k+=4UL )
1120 {
1121 const size_t i1( element->index() );
1122 const ET1 v1( element->value() );
1123 ++element;
1124 const size_t i2( element->index() );
1125 const ET1 v2( element->value() );
1126 ++element;
1127 const size_t i3( element->index() );
1128 const ET1 v3( element->value() );
1129 ++element;
1130 const size_t i4( element->index() );
1131 const ET1 v4( element->value() );
1132 ++element;
1133
1134 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1135
1136 C(i,j) += v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1137 }
1138
1139 for( ; element!=end; ++element )
1140 {
1141 const size_t i1( element->index() );
1142 const ET1 v1( element->value() );
1143
1144 C(i,j) += v1 * B(i1,j);
1145 }
1146 }
1147 }
1148 }
1149 }
1151 //**********************************************************************************************
1152
1153 //**Restructuring addition assignment***********************************************************
1168 template< typename MT // Type of the target matrix
1169 , bool SO > // Storage order of the target matrix
1170 friend inline auto addAssign( Matrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
1171 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1172 {
1174
1175 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1176 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1177
1178 const ForwardFunctor fwd;
1179
1180 addAssign( *lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1181 }
1183 //**********************************************************************************************
1184
1185 //**Addition assignment to sparse matrices******************************************************
1186 // No special implementation for the addition assignment to sparse matrices.
1187 //**********************************************************************************************
1188
1189 //**Subtraction assignment to dense matrices****************************************************
1202 template< typename MT // Type of the target dense matrix
1203 , bool SO > // Storage order of the target dense matrix
1204 friend inline auto subAssign( DenseMatrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
1205 -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1206 {
1208
1209 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1210 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1211
1212 LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side sparse matrix operand
1213 RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side dense matrix operand
1214
1215 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1216 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1217 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1218 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1219 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1220 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
1221
1222 SMatTDMatMultExpr::selectSubAssignKernel( *lhs, A, B );
1223 }
1225 //**********************************************************************************************
1226
1227 //**Default subtraction assignment to dense matrices********************************************
1241 template< typename MT3 // Type of the left-hand side target matrix
1242 , typename MT4 // Type of the left-hand side matrix operand
1243 , typename MT5 > // Type of the right-hand side matrix operand
1244 static inline auto selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1245 -> DisableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1246 {
1247 const size_t M( A.rows() );
1248 const size_t N( B.columns() );
1249
1250 BLAZE_INTERNAL_ASSERT( !( LOW || UPP ) || M == N, "Broken invariant detected" );
1251
1252 {
1253 size_t j( 0UL );
1254
1255 for( ; (j+4UL) <= N; j+=4UL ) {
1256 for( size_t i=( LOW ? j : 0UL ); i<( UPP ? j+4UL : M ); ++i )
1257 {
1258 const auto end( ( IsUpper_v<MT5> )
1259 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j+4UL) : A.upperBound(i,j+4UL) )
1260 :( A.end(i) ) );
1261 auto element( ( IsLower_v<MT5> )
1262 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
1263 :( A.begin(i) ) );
1264
1265 for( ; element!=end; ++element ) {
1266 C(i,j ) -= element->value() * B(element->index(),j );
1267 C(i,j+1UL) -= element->value() * B(element->index(),j+1UL);
1268 C(i,j+2UL) -= element->value() * B(element->index(),j+2UL);
1269 C(i,j+3UL) -= element->value() * B(element->index(),j+3UL);
1270 }
1271 }
1272 }
1273
1274 for( ; (j+2UL) <= N; j+=2UL ) {
1275 for( size_t i=( LOW ? j : 0UL ); i<( UPP ? j+2UL : M ); ++i )
1276 {
1277 const auto end( ( IsUpper_v<MT5> )
1278 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j+2UL) : A.upperBound(i,j+2UL) )
1279 :( A.end(i) ) );
1280 auto element( ( IsLower_v<MT5> )
1281 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
1282 :( A.begin(i) ) );
1283
1284 for( ; element!=end; ++element ) {
1285 C(i,j ) -= element->value() * B(element->index(),j );
1286 C(i,j+1UL) -= element->value() * B(element->index(),j+1UL);
1287 }
1288 }
1289 }
1290
1291 for( ; j<N; ++j ) {
1292 for( size_t i=( LOW ? j : 0UL ); i<( UPP ? j+1UL : M ); ++i )
1293 {
1294 const auto end( ( IsUpper_v<MT5> )
1295 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j) : A.upperBound(i,j) )
1296 :( A.end(i) ) );
1297 auto element( ( IsLower_v<MT5> )
1298 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
1299 :( A.begin(i) ) );
1300
1301 for( ; element!=end; ++element ) {
1302 C(i,j) -= element->value() * B(element->index(),j);
1303 }
1304 }
1305 }
1306 }
1307 }
1309 //**********************************************************************************************
1310
1311 //**Optimized subtraction assignment to dense matrices******************************************
1325 template< typename MT3 // Type of the left-hand side target matrix
1326 , typename MT4 // Type of the left-hand side matrix operand
1327 , typename MT5 > // Type of the right-hand side matrix operand
1328 static inline auto selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1329 -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1330 {
1331 const size_t M( A.rows() );
1332 const size_t N( B.columns() );
1333
1334 BLAZE_INTERNAL_ASSERT( !( LOW || UPP ) || M == N, "Broken invariant detected" );
1335
1336 {
1337 size_t j( 0UL );
1338
1339 for( ; (j+4UL) <= N; j+=4UL ) {
1340 for( size_t i=( LOW ? j : 0UL ); i<( UPP ? j+4UL : M ); ++i )
1341 {
1342 const auto end( ( IsUpper_v<MT5> )
1343 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j+4UL) : A.upperBound(i,j+4UL) )
1344 :( A.end(i) ) );
1345 auto element( ( IsLower_v<MT5> )
1346 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
1347 :( A.begin(i) ) );
1348
1349 const size_t nonzeros( end - element );
1350 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1351 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1352
1353 for( size_t k=0UL; k<kpos; k+=4UL )
1354 {
1355 const size_t i1( element->index() );
1356 const ET1 v1( element->value() );
1357 ++element;
1358 const size_t i2( element->index() );
1359 const ET1 v2( element->value() );
1360 ++element;
1361 const size_t i3( element->index() );
1362 const ET1 v3( element->value() );
1363 ++element;
1364 const size_t i4( element->index() );
1365 const ET1 v4( element->value() );
1366 ++element;
1367
1368 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1369
1370 C(i,j ) -= v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1371 C(i,j+1UL) -= v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1372 C(i,j+2UL) -= v1 * B(i1,j+2UL) + v2 * B(i2,j+2UL) + v3 * B(i3,j+2UL) + v4 * B(i4,j+2UL);
1373 C(i,j+3UL) -= v1 * B(i1,j+3UL) + v2 * B(i2,j+3UL) + v3 * B(i3,j+3UL) + v4 * B(i4,j+3UL);
1374 }
1375
1376 for( ; element!=end; ++element )
1377 {
1378 const size_t i1( element->index() );
1379 const ET1 v1( element->value() );
1380
1381 C(i,j ) -= v1 * B(i1,j );
1382 C(i,j+1UL) -= v1 * B(i1,j+1UL);
1383 C(i,j+2UL) -= v1 * B(i1,j+2UL);
1384 C(i,j+3UL) -= v1 * B(i1,j+3UL);
1385 }
1386 }
1387 }
1388
1389 for( ; (j+2UL) <= N; j+=2UL ) {
1390 for( size_t i=( LOW ? j : 0UL ); i<( UPP ? j+2UL : M ); ++i )
1391 {
1392 const auto end( ( IsUpper_v<MT5> )
1393 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j+2UL) : A.upperBound(i,j+2UL) )
1394 :( A.end(i) ) );
1395 auto element( ( IsLower_v<MT5> )
1396 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
1397 :( A.begin(i) ) );
1398
1399 const size_t nonzeros( end - element );
1400 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1401 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1402
1403 for( size_t k=0UL; k<kpos; k+=4UL )
1404 {
1405 const size_t i1( element->index() );
1406 const ET1 v1( element->value() );
1407 ++element;
1408 const size_t i2( element->index() );
1409 const ET1 v2( element->value() );
1410 ++element;
1411 const size_t i3( element->index() );
1412 const ET1 v3( element->value() );
1413 ++element;
1414 const size_t i4( element->index() );
1415 const ET1 v4( element->value() );
1416 ++element;
1417
1418 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1419
1420 C(i,j ) -= v1 * B(i1,j ) + v2 * B(i2,j ) + v3 * B(i3,j ) + v4 * B(i4,j );
1421 C(i,j+1UL) -= v1 * B(i1,j+1UL) + v2 * B(i2,j+1UL) + v3 * B(i3,j+1UL) + v4 * B(i4,j+1UL);
1422 }
1423
1424 for( ; element!=end; ++element )
1425 {
1426 const size_t i1( element->index() );
1427 const ET1 v1( element->value() );
1428
1429 C(i,j ) -= v1 * B(i1,j );
1430 C(i,j+1UL) -= v1 * B(i1,j+1UL);
1431 }
1432 }
1433 }
1434
1435 for( ; j<N; ++j ) {
1436 for( size_t i=( LOW ? j : 0UL ); i<( UPP ? j+1UL : M ); ++i )
1437 {
1438 const auto end( ( IsUpper_v<MT5> )
1439 ?( IsStrictlyUpper_v<MT5> ? A.lowerBound(i,j) : A.upperBound(i,j) )
1440 :( A.end(i) ) );
1441 auto element( ( IsLower_v<MT5> )
1442 ?( IsStrictlyLower_v<MT5> ? A.upperBound(i,j) : A.lowerBound(i,j) )
1443 :( A.begin(i) ) );
1444
1445 const size_t nonzeros( end - element );
1446 const size_t kpos( prevMultiple( nonzeros, 4UL ) );
1447 BLAZE_INTERNAL_ASSERT( kpos <= nonzeros, "Invalid end calculation" );
1448
1449 for( size_t k=0UL; k<kpos; k+=4UL )
1450 {
1451 const size_t i1( element->index() );
1452 const ET1 v1( element->value() );
1453 ++element;
1454 const size_t i2( element->index() );
1455 const ET1 v2( element->value() );
1456 ++element;
1457 const size_t i3( element->index() );
1458 const ET1 v3( element->value() );
1459 ++element;
1460 const size_t i4( element->index() );
1461 const ET1 v4( element->value() );
1462 ++element;
1463
1464 BLAZE_INTERNAL_ASSERT( i1 < i2 && i2 < i3 && i3 < i4, "Invalid sparse matrix index detected" );
1465
1466 C(i,j) -= v1 * B(i1,j) + v2 * B(i2,j) + v3 * B(i3,j) + v4 * B(i4,j);
1467 }
1468
1469 for( ; element!=end; ++element )
1470 {
1471 const size_t i1( element->index() );
1472 const ET1 v1( element->value() );
1473
1474 C(i,j) -= v1 * B(i1,j);
1475 }
1476 }
1477 }
1478 }
1479 }
1481 //**********************************************************************************************
1482
1483 //**Restructuring subtraction assignment********************************************************
1498 template< typename MT // Type of the target matrix
1499 , bool SO > // Storage order of the target matrix
1500 friend inline auto subAssign( Matrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
1501 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1502 {
1504
1505 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1506 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1507
1508 const ForwardFunctor fwd;
1509
1510 subAssign( *lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1511 }
1513 //**********************************************************************************************
1514
1515 //**Subtraction assignment to sparse matrices***************************************************
1516 // No special implementation for the subtraction assignment to sparse matrices.
1517 //**********************************************************************************************
1518
1519 //**Schur product assignment to dense matrices**************************************************
1532 template< typename MT // Type of the target dense matrix
1533 , bool SO > // Storage order of the target dense matrix
1534 friend inline void schurAssign( DenseMatrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
1535 {
1537
1541
1542 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1543 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1544
1545 const ResultType tmp( serial( rhs ) );
1546 schurAssign( *lhs, tmp );
1547 }
1549 //**********************************************************************************************
1550
1551 //**Schur product assignment to sparse matrices*************************************************
1552 // No special implementation for the Schur product assignment to sparse matrices.
1553 //**********************************************************************************************
1554
1555 //**Multiplication assignment to dense matrices*************************************************
1556 // No special implementation for the multiplication assignment to dense matrices.
1557 //**********************************************************************************************
1558
1559 //**Multiplication assignment to sparse matrices************************************************
1560 // No special implementation for the multiplication assignment to sparse matrices.
1561 //**********************************************************************************************
1562
1563 //**SMP assignment to dense matrices************************************************************
1578 template< typename MT // Type of the target dense matrix
1579 , bool SO > // Storage order of the target dense matrix
1580 friend inline auto smpAssign( DenseMatrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
1581 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1582 {
1584
1585 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1586 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1587
1588 LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1589 RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1590
1591 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1592 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1593 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1594 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1595 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1596 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
1597
1598 smpAssign( *lhs, A * B );
1599 }
1601 //**********************************************************************************************
1602
1603 //**SMP assignment to sparse matrices***********************************************************
1618 template< typename MT // Type of the target sparse matrix
1619 , bool SO > // Storage order of the target sparse matrix
1620 friend inline auto smpAssign( SparseMatrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
1621 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1622 {
1624
1625 using TmpType = If_t< SO, ResultType, OppositeType >;
1626
1633
1634 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1635 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1636
1637 const ForwardFunctor fwd;
1638
1639 const TmpType tmp( rhs );
1640 smpAssign( *lhs, fwd( tmp ) );
1641 }
1643 //**********************************************************************************************
1644
1645 //**Restructuring SMP assignment****************************************************************
1660 template< typename MT // Type of the target matrix
1661 , bool SO > // Storage order of the target matrix
1662 friend inline auto smpAssign( Matrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
1663 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1664 {
1666
1667 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1668 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1669
1670 const ForwardFunctor fwd;
1671
1672 smpAssign( *lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1673 }
1675 //**********************************************************************************************
1676
1677 //**SMP addition assignment to dense matrices***************************************************
1693 template< typename MT // Type of the target dense matrix
1694 , bool SO > // Storage order of the target dense matrix
1695 friend inline auto smpAddAssign( DenseMatrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
1696 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1697 {
1699
1700 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1701 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1702
1703 LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1704 RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1705
1706 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1707 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1708 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1709 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1710 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1711 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
1712
1713 smpAddAssign( *lhs, A * B );
1714 }
1716 //**********************************************************************************************
1717
1718 //**Restructuring SMP addition assignment*******************************************************
1733 template< typename MT // Type of the target matrix
1734 , bool SO > // Storage order of the target matrix
1735 friend inline auto smpAddAssign( Matrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
1736 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1737 {
1739
1740 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1741 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1742
1743 const ForwardFunctor fwd;
1744
1745 smpAddAssign( *lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1746 }
1748 //**********************************************************************************************
1749
1750 //**SMP addition assignment to sparse matrices**************************************************
1751 // No special implementation for the SMP addition assignment to sparse matrices.
1752 //**********************************************************************************************
1753
1754 //**SMP subtraction assignment to dense matrices************************************************
1770 template< typename MT // Type of the target dense matrix
1771 , bool SO > // Storage order of the target dense matrix
1772 friend inline auto smpSubAssign( DenseMatrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
1773 -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1774 {
1776
1777 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1778 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1779
1780 LT A( rhs.lhs_ ); // Evaluation of the left-hand side sparse matrix operand
1781 RT B( rhs.rhs_ ); // Evaluation of the right-hand side dense matrix operand
1782
1783 BLAZE_INTERNAL_ASSERT( A.rows() == rhs.lhs_.rows() , "Invalid number of rows" );
1784 BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
1785 BLAZE_INTERNAL_ASSERT( B.rows() == rhs.rhs_.rows() , "Invalid number of rows" );
1786 BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
1787 BLAZE_INTERNAL_ASSERT( A.rows() == (*lhs).rows() , "Invalid number of rows" );
1788 BLAZE_INTERNAL_ASSERT( B.columns() == (*lhs).columns() , "Invalid number of columns" );
1789
1790 smpSubAssign( *lhs, A * B );
1791 }
1793 //**********************************************************************************************
1794
1795 //**Restructuring SMP subtraction assignment****************************************************
1810 template< typename MT // Type of the target matrix
1811 , bool SO > // Storage order of the target matrix
1812 friend inline auto smpSubAssign( Matrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
1813 -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1814 {
1816
1817 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1818 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1819
1820 const ForwardFunctor fwd;
1821
1822 smpSubAssign( *lhs, fwd( rhs.lhs_ * trans( rhs.rhs_ ) ) );
1823 }
1825 //**********************************************************************************************
1826
1827 //**SMP subtraction assignment to sparse matrices***********************************************
1828 // No special implementation for the SMP subtraction assignment to sparse matrices.
1829 //**********************************************************************************************
1830
1831 //**SMP Schur product assignment to dense matrices**********************************************
1844 template< typename MT // Type of the target dense matrix
1845 , bool SO > // Storage order of the target dense matrix
1846 friend inline void smpSchurAssign( DenseMatrix<MT,SO>& lhs, const SMatTDMatMultExpr& rhs )
1847 {
1849
1853
1854 BLAZE_INTERNAL_ASSERT( (*lhs).rows() == rhs.rows() , "Invalid number of rows" );
1855 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == rhs.columns(), "Invalid number of columns" );
1856
1857 const ResultType tmp( rhs );
1858 smpSchurAssign( *lhs, tmp );
1859 }
1861 //**********************************************************************************************
1862
1863 //**SMP Schur product assignment to sparse matrices*********************************************
1864 // No special implementation for the SMP Schur product assignment to sparse matrices.
1865 //**********************************************************************************************
1866
1867 //**SMP multiplication assignment to dense matrices*********************************************
1868 // No special implementation for the SMP multiplication assignment to dense matrices.
1869 //**********************************************************************************************
1870
1871 //**SMP multiplication assignment to sparse matrices********************************************
1872 // No special implementation for the SMP multiplication assignment to sparse matrices.
1873 //**********************************************************************************************
1874
1875 //**Compile time checks*************************************************************************
1884 //**********************************************************************************************
1885};
1886//*************************************************************************************************
1887
1888
1889
1890
1891//=================================================================================================
1892//
1893// GLOBAL BINARY ARITHMETIC OPERATORS
1894//
1895//=================================================================================================
1896
1897//*************************************************************************************************
1910template< typename MT1 // Type of the left-hand side dense matrix
1911 , typename MT2 // Type of the right-hand side sparse matrix
1912 , DisableIf_t< ( IsIdentity_v<MT1> &&
1913 IsSame_v< ElementType_t<MT1>, ElementType_t<MT2> > ) ||
1914 IsZero_v<MT1> >* = nullptr >
1915inline const SMatTDMatMultExpr<MT1,MT2,false,false,false,false>
1916 smattdmatmult( const SparseMatrix<MT1,false>& lhs, const DenseMatrix<MT2,true>& rhs )
1917{
1919
1920 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == (*rhs).rows(), "Invalid matrix sizes" );
1921
1922 return SMatTDMatMultExpr<MT1,MT2,false,false,false,false>( *lhs, *rhs );
1923}
1925//*************************************************************************************************
1926
1927
1928//*************************************************************************************************
1942template< typename MT1 // Type of the left-hand side sparse matrix
1943 , typename MT2 // Type of the right-hand side dense matrix
1944 , EnableIf_t< IsIdentity_v<MT1> &&
1945 IsSame_v< ElementType_t<MT1>, ElementType_t<MT2> > >* = nullptr >
1946inline const MT2&
1947 smattdmatmult( const SparseMatrix<MT1,false>& lhs, const DenseMatrix<MT2,true>& rhs )
1948{
1950
1951 MAYBE_UNUSED( lhs );
1952
1953 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == (*rhs).rows(), "Invalid matrix sizes" );
1954
1955 return (*rhs);
1956}
1958//*************************************************************************************************
1959
1960
1961//*************************************************************************************************
1974template< typename MT1 // Type of the left-hand side dense matrix
1975 , typename MT2 // Type of the right-hand side sparse matrix
1976 , EnableIf_t< IsZero_v<MT1> >* = nullptr >
1977inline decltype(auto)
1978 smattdmatmult( const SparseMatrix<MT1,false>& lhs, const DenseMatrix<MT2,true>& rhs )
1979{
1981
1982 BLAZE_INTERNAL_ASSERT( (*lhs).columns() == (*rhs).rows(), "Invalid matrix sizes" );
1983
1984 using ReturnType = const MultTrait_t< ResultType_t<MT1>, ResultType_t<MT2> >;
1985
1988
1989 return ReturnType( (*lhs).rows(), (*rhs).columns() );
1990}
1992//*************************************************************************************************
1993
1994
1995//*************************************************************************************************
2026template< typename MT1 // Type of the left-hand side sparse matrix
2027 , typename MT2 > // Type of the right-hand side dense matrix
2028inline decltype(auto)
2029 operator*( const SparseMatrix<MT1,false>& lhs, const DenseMatrix<MT2,true>& rhs )
2030{
2032
2033 if( (*lhs).columns() != (*rhs).rows() ) {
2034 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
2035 }
2036
2037 return smattdmatmult( *lhs, *rhs );
2038}
2039//*************************************************************************************************
2040
2041
2042
2043
2044//=================================================================================================
2045//
2046// GLOBAL FUNCTIONS
2047//
2048//=================================================================================================
2049
2050//*************************************************************************************************
2076template< typename MT1 // Type of the left-hand side dense matrix
2077 , typename MT2 // Type of the right-hand side dense matrix
2078 , bool SF // Symmetry flag
2079 , bool HF // Hermitian flag
2080 , bool LF // Lower flag
2081 , bool UF > // Upper flag
2082inline decltype(auto) declsym( const SMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2083{
2085
2086 if( !isSquare( dm ) ) {
2087 BLAZE_THROW_INVALID_ARGUMENT( "Invalid symmetric matrix specification" );
2088 }
2089
2090 using ReturnType = const SMatTDMatMultExpr<MT1,MT2,true,HF,LF,UF>;
2091 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2092}
2094//*************************************************************************************************
2095
2096
2097//*************************************************************************************************
2123template< typename MT1 // Type of the left-hand side dense matrix
2124 , typename MT2 // Type of the right-hand side dense matrix
2125 , bool SF // Symmetry flag
2126 , bool HF // Hermitian flag
2127 , bool LF // Lower flag
2128 , bool UF > // Upper flag
2129inline decltype(auto) declherm( const SMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2130{
2132
2133 if( !isSquare( dm ) ) {
2134 BLAZE_THROW_INVALID_ARGUMENT( "Invalid Hermitian matrix specification" );
2135 }
2136
2137 using ReturnType = const SMatTDMatMultExpr<MT1,MT2,SF,true,LF,UF>;
2138 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2139}
2141//*************************************************************************************************
2142
2143
2144//*************************************************************************************************
2170template< typename MT1 // Type of the left-hand side dense matrix
2171 , typename MT2 // Type of the right-hand side dense matrix
2172 , bool SF // Symmetry flag
2173 , bool HF // Hermitian flag
2174 , bool LF // Lower flag
2175 , bool UF > // Upper flag
2176inline decltype(auto) decllow( const SMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2177{
2179
2180 if( !isSquare( dm ) ) {
2181 BLAZE_THROW_INVALID_ARGUMENT( "Invalid lower matrix specification" );
2182 }
2183
2184 using ReturnType = const SMatTDMatMultExpr<MT1,MT2,SF,HF,true,UF>;
2185 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2186}
2188//*************************************************************************************************
2189
2190
2191//*************************************************************************************************
2217template< typename MT1 // Type of the left-hand side dense matrix
2218 , typename MT2 // Type of the right-hand side dense matrix
2219 , bool SF // Symmetry flag
2220 , bool HF // Hermitian flag
2221 , bool UF > // Upper flag
2222inline decltype(auto) declunilow( const SMatTDMatMultExpr<MT1,MT2,SF,HF,false,UF>& dm )
2223{
2225
2226 if( !isSquare( dm ) ) {
2227 BLAZE_THROW_INVALID_ARGUMENT( "Invalid unilower matrix specification" );
2228 }
2229
2230 return declunilow( decllow( *dm ) );
2231}
2233//*************************************************************************************************
2234
2235
2236//*************************************************************************************************
2262template< typename MT1 // Type of the left-hand side dense matrix
2263 , typename MT2 // Type of the right-hand side dense matrix
2264 , bool SF // Symmetry flag
2265 , bool HF // Hermitian flag
2266 , bool UF > // Upper flag
2267inline decltype(auto) declstrlow( const SMatTDMatMultExpr<MT1,MT2,SF,HF,false,UF>& dm )
2268{
2270
2271 if( !isSquare( dm ) ) {
2272 BLAZE_THROW_INVALID_ARGUMENT( "Invalid strictly lower matrix specification" );
2273 }
2274
2275 return declstrlow( decllow( *dm ) );
2276}
2278//*************************************************************************************************
2279
2280
2281//*************************************************************************************************
2307template< typename MT1 // Type of the left-hand side dense matrix
2308 , typename MT2 // Type of the right-hand side dense matrix
2309 , bool SF // Symmetry flag
2310 , bool HF // Hermitian flag
2311 , bool LF // Lower flag
2312 , bool UF > // Upper flag
2313inline decltype(auto) declupp( const SMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2314{
2316
2317 if( !isSquare( dm ) ) {
2318 BLAZE_THROW_INVALID_ARGUMENT( "Invalid upper matrix specification" );
2319 }
2320
2321 using ReturnType = const SMatTDMatMultExpr<MT1,MT2,SF,HF,LF,true>;
2322 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2323}
2325//*************************************************************************************************
2326
2327
2328//*************************************************************************************************
2354template< typename MT1 // Type of the left-hand side dense matrix
2355 , typename MT2 // Type of the right-hand side dense matrix
2356 , bool SF // Symmetry flag
2357 , bool HF // Hermitian flag
2358 , bool LF > // Lower flag
2359inline decltype(auto) decluniupp( const SMatTDMatMultExpr<MT1,MT2,SF,HF,LF,false>& dm )
2360{
2362
2363 if( !isSquare( dm ) ) {
2364 BLAZE_THROW_INVALID_ARGUMENT( "Invalid uniupper matrix specification" );
2365 }
2366
2367 return decluniupp( declupp( *dm ) );
2368}
2370//*************************************************************************************************
2371
2372
2373//*************************************************************************************************
2399template< typename MT1 // Type of the left-hand side dense matrix
2400 , typename MT2 // Type of the right-hand side dense matrix
2401 , bool SF // Symmetry flag
2402 , bool HF // Hermitian flag
2403 , bool LF > // Lower flag
2404inline decltype(auto) declstrupp( const SMatTDMatMultExpr<MT1,MT2,SF,HF,LF,false>& dm )
2405{
2407
2408 if( !isSquare( dm ) ) {
2409 BLAZE_THROW_INVALID_ARGUMENT( "Invalid strictly upper matrix specification" );
2410 }
2411
2412 return declstrupp( declupp( *dm ) );
2413}
2415//*************************************************************************************************
2416
2417
2418//*************************************************************************************************
2444template< typename MT1 // Type of the left-hand side dense matrix
2445 , typename MT2 // Type of the right-hand side dense matrix
2446 , bool SF // Symmetry flag
2447 , bool HF // Hermitian flag
2448 , bool LF // Lower flag
2449 , bool UF > // Upper flag
2450inline decltype(auto) decldiag( const SMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2451{
2453
2454 if( !isSquare( dm ) ) {
2455 BLAZE_THROW_INVALID_ARGUMENT( "Invalid diagonal matrix specification" );
2456 }
2457
2458 using ReturnType = const SMatTDMatMultExpr<MT1,MT2,SF,HF,true,true>;
2459 return ReturnType( dm.leftOperand(), dm.rightOperand() );
2460}
2462//*************************************************************************************************
2463
2464
2465
2466
2467//=================================================================================================
2468//
2469// SIZE SPECIALIZATIONS
2470//
2471//=================================================================================================
2472
2473//*************************************************************************************************
2475template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2476struct Size< SMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 0UL >
2477 : public Size<MT1,0UL>
2478{};
2479
2480template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2481struct Size< SMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 1UL >
2482 : public Size<MT2,1UL>
2483{};
2485//*************************************************************************************************
2486
2487
2488
2489
2490//=================================================================================================
2491//
2492// ISALIGNED SPECIALIZATIONS
2493//
2494//=================================================================================================
2495
2496//*************************************************************************************************
2498template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2499struct IsAligned< SMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2500 : public IsAligned<MT2>
2501{};
2503//*************************************************************************************************
2504
2505} // namespace blaze
2506
2507#endif
Header file for auxiliary alias declarations.
typename T::CompositeType CompositeType_t
Alias declaration for nested CompositeType type definitions.
Definition: Aliases.h:110
typename T::ResultType ResultType_t
Alias declaration for nested ResultType type definitions.
Definition: Aliases.h:450
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.
Definition: Aliases.h:190
typename T::OppositeType OppositeType_t
Alias declaration for nested OppositeType type definitions.
Definition: Aliases.h:310
typename T::TransposeType TransposeType_t
Alias declaration for nested TransposeType type definitions.
Definition: Aliases.h:550
Header file for run time assertion macros.
Header file for the blaze::checked and blaze::unchecked instances.
Constraints on the storage order of matrix types.
Header file for the conjugate shim.
Header file for the decldiag trait.
Header file for the DeclDiag functor.
Header file for the declherm trait.
Header file for the DeclHerm functor.
Header file for the decllow trait.
Header file for the DeclLow functor.
Header file for the declsym trait.
Header file for the DeclSym functor.
Header file for the declupp trait.
Header file for the DeclUpp functor.
Header file for the EnableIf class template.
Header file for the function trace functionality.
Header file for the If class template.
Header file for the IntegralConstant class template.
Header file for the IsAligned type trait.
Header file for the IsBuiltin type trait.
Header file for the IsComputation type trait class.
Header file for the IsDiagonal type trait.
Header file for the IsExpression type trait class.
Header file for the IsIdentity type trait.
Header file for the IsLower type trait.
Header file for the IsResizable type trait.
Header file for the IsStrictlyLower type trait.
Header file for the IsStrictlyUpper type trait.
Header file for the IsSymmetric type trait.
Header file for the IsTriangular type trait.
Header file for the IsUpper type trait.
Header file for the MAYBE_UNUSED function template.
Header file for the multiplication trait.
Header file for the Noop functor.
Header file for the prevMultiple shim.
Constraints on the storage order of matrix types.
Constraint on the data type.
Base class for dense matrices.
Definition: DenseMatrix.h:82
Expression object for sparse matrix-transpose dense matrix multiplications.
Definition: SMatTDMatMultExpr.h:127
ElementType_t< RT1 > ET1
Element type of the left-hand side dense matrix expression.
Definition: SMatTDMatMultExpr.h:132
CompositeType_t< MT1 > CT1
Composite type of the left-hand side sparse matrix expression.
Definition: SMatTDMatMultExpr.h:134
TransposeType_t< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: SMatTDMatMultExpr.h:234
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: SMatTDMatMultExpr.h:411
If_t< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition: SMatTDMatMultExpr.h:249
static constexpr bool UPP
Flag for upper matrices.
Definition: SMatTDMatMultExpr.h:152
RightOperand rhs_
Right-hand side dense matrix of the multiplication expression.
Definition: SMatTDMatMultExpr.h:429
LeftOperand lhs_
Left-hand side sparse matrix of the multiplication expression.
Definition: SMatTDMatMultExpr.h:428
If_t< IsExpression_v< MT2 >, const MT2, const MT2 & > RightOperand
Composite type of the right-hand side dense matrix expression.
Definition: SMatTDMatMultExpr.h:243
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: SMatTDMatMultExpr.h:389
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: SMatTDMatMultExpr.h:421
static constexpr bool evaluateRight
Compilation switch for the composite type of the right-hand side dense matrix expression.
Definition: SMatTDMatMultExpr.h:145
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: SMatTDMatMultExpr.h:401
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: SMatTDMatMultExpr.h:282
static constexpr bool smpAssignable
Compilation switch for the expression template assignment strategy.
Definition: SMatTDMatMultExpr.h:257
CompositeType_t< MT2 > CT2
Composite type of the right-hand side dense matrix expression.
Definition: SMatTDMatMultExpr.h:135
const ResultType CompositeType
Data type for composite expression templates.
Definition: SMatTDMatMultExpr.h:237
static constexpr bool LOW
Flag for lower matrices.
Definition: SMatTDMatMultExpr.h:151
static constexpr bool HERM
Flag for Hermitian matrices.
Definition: SMatTDMatMultExpr.h:150
OppositeType_t< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: SMatTDMatMultExpr.h:233
ElementType_t< RT2 > ET2
Element type of the right-hand side sparse matrix expression.
Definition: SMatTDMatMultExpr.h:133
static constexpr bool SYM
Flag for symmetric matrices.
Definition: SMatTDMatMultExpr.h:149
static constexpr bool simdEnabled
Compilation switch for the expression template evaluation strategy.
Definition: SMatTDMatMultExpr.h:254
typename If_t< HERM, DeclHermTrait< MultTrait_t< RT1, RT2 > >, If_t< SYM, DeclSymTrait< MultTrait_t< RT1, RT2 > >, If_t< LOW, If_t< UPP, DeclDiagTrait< MultTrait_t< RT1, RT2 > >, DeclLowTrait< MultTrait_t< RT1, RT2 > > >, If_t< UPP, DeclUppTrait< MultTrait_t< RT1, RT2 > >, MultTrait< RT1, RT2 > > > > >::Type ResultType
Result type for expression template evaluations.
Definition: SMatTDMatMultExpr.h:231
If_t< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side sparse matrix operand.
Definition: SMatTDMatMultExpr.h:246
ElementType_t< ResultType > ElementType
Resulting element type.
Definition: SMatTDMatMultExpr.h:235
If_t< IsExpression_v< MT1 >, const MT1, const MT1 & > LeftOperand
Composite type of the left-hand side sparse matrix expression.
Definition: SMatTDMatMultExpr.h:240
static constexpr bool evaluateLeft
Compilation switch for the composite type of the left-hand side sparse matrix expression.
Definition: SMatTDMatMultExpr.h:140
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: SMatTDMatMultExpr.h:331
ResultType_t< MT1 > RT1
Result type of the left-hand side sparse matrix expression.
Definition: SMatTDMatMultExpr.h:130
ResultType_t< MT2 > RT2
Result type of the right-hand side dense matrix expression.
Definition: SMatTDMatMultExpr.h:131
const ElementType ReturnType
Return type for expression template evaluations.
Definition: SMatTDMatMultExpr.h:236
LeftOperand leftOperand() const noexcept
Returns the left-hand side sparse matrix operand.
Definition: SMatTDMatMultExpr.h:367
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: SMatTDMatMultExpr.h:347
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: SMatTDMatMultExpr.h:357
SMatTDMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the SMatTDMatMultExpr class.
Definition: SMatTDMatMultExpr.h:267
RightOperand rightOperand() const noexcept
Returns the right-hand side transpose dense matrix operand.
Definition: SMatTDMatMultExpr.h:377
Base class for sparse matrices.
Definition: SparseMatrix.h:77
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Header file for the Computation base class.
Header file for the DenseMatrix base class.
Header file for the MatMatMultExpr base class.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:137
decltype(auto) min(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise minimum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1339
decltype(auto) max(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise maximum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1375
decltype(auto) declstrupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as strictly upper.
Definition: DMatDeclStrUppExpr.h:1003
decltype(auto) conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition: DMatMapExpr.h:1464
decltype(auto) decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as diagonal.
Definition: DMatDeclDiagExpr.h:978
decltype(auto) trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:766
decltype(auto) declstrlow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as strictly lower.
Definition: DMatDeclStrLowExpr.h:1003
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:812
decltype(auto) declupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as upper.
Definition: DMatDeclUppExpr.h:1004
decltype(auto) decllow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as lower.
Definition: DMatDeclLowExpr.h:1004
decltype(auto) decluniupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as uniupper.
Definition: DMatDeclUniUppExpr.h:1005
decltype(auto) declherm(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as Hermitian.
Definition: DMatDeclHermExpr.h:1005
decltype(auto) declsym(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as symmetric.
Definition: DMatDeclSymExpr.h:1005
decltype(auto) declunilow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as unilower.
Definition: DMatDeclUniLowExpr.h:1004
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition: RowMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.
Definition: StorageOrder.h:84
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.
Definition: RequiresEvaluation.h:81
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.
Definition: MatMatMultExpr.h:103
#define BLAZE_CONSTRAINT_MUST_NOT_BE_ZERO_TYPE(T)
Constraint on the data type.
Definition: Zero.h:81
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.
Definition: DenseMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.
Definition: SparseMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition: ColumnMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_ZERO_TYPE(T)
Constraint on the data type.
Definition: Zero.h:61
BLAZE_ALWAYS_INLINE constexpr auto prevMultiple(T1 value, T2 factor) noexcept
Rounds down an integral value to the previous multiple of a given factor.
Definition: PrevMultiple.h:68
constexpr void reset(Matrix< MT, SO > &matrix)
Resetting the given matrix.
Definition: Matrix.h:806
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:584
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:518
bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition: Matrix.h:1383
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:137
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.
Definition: Assert.h:101
auto smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:162
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
auto smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:194
auto smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:131
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:158
constexpr void MAYBE_UNUSED(const Args &...)
Suppression of unused parameter warnings.
Definition: MaybeUnused.h:81
typename If< Condition >::template Type< T1, T2 > If_t
Auxiliary alias template for the If class template.
Definition: If.h:108
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.
Definition: Exception.h:331
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.
Definition: Exception.h:235
typename EnableIf<!Condition, T >::Type DisableIf_t
Auxiliary type for the EnableIf class template.
Definition: EnableIf.h:175
#define BLAZE_FUNCTION_TRACE
Function trace macro.
Definition: FunctionTrace.h:94
constexpr Unchecked unchecked
Global Unchecked instance.
Definition: Check.h:146
Header file for the exception macros of the math module.
Constraints on the storage order of matrix types.
Header file for all forward declarations for expression class templates.
Header file for the Size type trait.
Header file for the reset shim.
Header file for the serial shim.
Base class for all compute expression templates.
Definition: Computation.h:68
Base template for the DeclDiagTrait class.
Definition: DeclDiagTrait.h:127
Generic wrapper for the decldiag() function.
Definition: DeclDiag.h:61
Base template for the DeclHermTrait class.
Definition: DeclHermTrait.h:126
Generic wrapper for the declherm() function.
Definition: DeclHerm.h:61
Base template for the DeclLowTrait class.
Definition: DeclLowTrait.h:126
Generic wrapper for the decllow() function.
Definition: DeclLow.h:61
Base template for the DeclSymTrait class.
Definition: DeclSymTrait.h:126
Generic wrapper for the declsym() function.
Definition: DeclSym.h:61
Base template for the DeclUppTrait class.
Definition: DeclUppTrait.h:126
Generic wrapper for the declupp() function.
Definition: DeclUpp.h:61
Base class for all matrix/matrix multiplication expression templates.
Definition: MatMatMultExpr.h:71
Base template for the MultTrait class.
Definition: MultTrait.h:130
Generic wrapper for the null function.
Definition: Noop.h:62
System settings for performance optimizations.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
Header file for the IsZero type trait.
Header file for the RequiresEvaluation type trait.
Header file for basic type definitions.
Header file for the generic max algorithm.
Header file for the generic min algorithm.