spark 矩阵转置 java_Spark MLlib之使用Breeze操作矩阵向量

冀弘济

2023-12-01

这下面的练习中，需要自己将spark的jar包添加进来。

1.spark Mlib 底层使用的向量、矩阵运算使用了Breeze库。

scalaNLP 是一套机器学习和数值技算的库。它主要是关于科学技术(sc)、机器学习(ML)和自然语言处理(NLP)的。它包括三个库，Breeze、Epic 和 Puck。

Breeze :是机器学习和数值技术库，它是sparkMlib的核心，包括线性代数、数值技术和优化，是一种通用、功能强大、有效的机器学习方法。

Epic :是一种高性能能统计分析器和结构化预测库

Puck :是一个快速GPU加速解析器

在使用Breeze 库时，需要导入相关包：

import breeze.linalg._

import breeze.numerics._

具体练习如下：

package leaning

import breeze.linalg._

import breeze.numerics._

import breeze.stats.distributions.Rand

/**

* Created by dy9776 on 2017/12/5.

object Practise_breeze{

def main(args: Array[String]) {

val matrix: DenseMatrix[Double] = DenseMatrix.zeros[Double](3,2)

println(matrix)

0.0 0.0

//全0向量

val testVector: DenseVector[Double] = DenseVector.zeros[Double](2)

println(testVector)

//全1向量

val allOneVector=DenseVector.ones[Double](2)

println(allOneVector)

//按数值填充向量

val haveNumberFill =DenseVector.fill[Double](3,2)

println(haveNumberFill)

//生成随机向量

val rangeNUm= DenseVector.range(1, 10 , 2)//DenseVector(1, 3, 5, 7, 9)

val rangeNUmD= DenseVector.rangeD(1, 9 , 2)//DenseVector(1.0, 3.0, 5.0, 7.0)

val rangeNUmF= DenseVector.rangeF(1, 7 , 2)//DenseVector(1.0, 3.0, 5.0)

println(rangeNUm)

println(rangeNUmD)

println(rangeNUmF)

//单位矩阵

val unitMatrix=DenseMatrix.eye[Double](4)

// println(unitMatrix)

1.0 0.0 0.0 0.0

0.0 1.0 0.0 0.0

0.0 0.0 1.0 0.0

0.0 0.0 0.0 1.0

//对角矩阵

val doubleVecoter=diag(DenseVector(3.0, 4.0 , 5.0))

// println(doubleVecoter)

3.0 0.0 0.0

0.0 4.0 0.0

0.0 0.0 5.0

//按照行创建矩阵

val byRowCreateMatrix= DenseMatrix( (4.0, 5.0, 6.0 ) , (7.0 ,8.0 ,9.0))

// println(byRowCreateMatrix)

4.0 5.0 6.0

7.0 8.0 9.0

//按照行创建向量

val denseCreateVector = DenseVector((4.0, 5.0, 6.0, 7.0, 8.0, 9.0))

// println(denseCreateVector) ///DenseVector((4.0,5.0,6.0,7.0,8.0,9.0)

//向量装置

val vectorTranspostion= DenseVector( (4.0, 5.0, 6.0, 7.0, 8.0, 9.0) ).t

println(vectorTranspostion)//Transpose(DenseVector((4.0,5.0,6.0,7.0,8.0,9.0)))

//从函数创建向量

val funCreateVector=DenseVector.tabulate(5)(i=> i*i)

println(funCreateVector)//DenseVector(0, 1, 4, 9, 16)

val funCreateVector2=DenseVector.tabulate( 0 to 5)(i=> i*i)

println(funCreateVector2)//DenseVector(0, 1, 4, 9, 16, 25)

//从函数创建矩阵

val createFuncMatrix= DenseMatrix.tabulate(3, 4) {

case (i ,j ) => i*i + j*j

}

// println(createFuncMatrix)

0 1 4 9

1 2 5 10

4 5 8 13

//从数组创建矩阵

val createFunctionMatrix= new DenseMatrix[Double](3, 2, Array(1.0, 4.0, 7.0, 3.0, 6.0, 9.0))

// println(createFunctionMatrix)

1.0 3.0

4.0 6.0

7.0 9.0

//0 到 1的随机向量

val formZeroToOneRandomVector= DenseVector.rand( 9, Rand.uniform)

println(formZeroToOneRandomVector)

// DenseVector(0.7978222133507369, 0.48978247271729325, 0.24943434133065834, 0.6619447026155139, 0.5324942068054981, 0.9051865626036415, 0.5989291014099107, 0.9221881029987078, 0.17371486701192662)

val formZeroToOneRandomVector2= DenseVector.rand( 9, Rand.uniform)

println(formZeroToOneRandomVector2)

//DenseVector(0.7978222133507369, 0.48978247271729325, 0.24943434133065834, 0.6619447026155139, 0.5324942068054981, 0.9051865626036415, 0.5989291014099107, 0.9221881029987078, 0.17371486701192662)

//0 到 1 的随机矩阵

val formZeroToOneRandomMatrix= DenseMatrix.rand(3, 2, Rand.uniform)

println(formZeroToOneRandomMatrix)

0.8036324612618653 0.538112087890035

0.6864375371630702 0.3123993272549075

0.9458628172312897 0.01137554621536796

val formZeroToOneRandomMatrix2=DenseMatrix.rand(3, 2, Rand.gaussian)

println(formZeroToOneRandomMatrix2)

0.9510499901472648 0.287812938654061

-0.5266499883462216 0.9380426076781263

-0.3959295333472151 -0.9057610233257112

//Breeze元素访问

val a = new DenseVector[Int](Array(1 to 20 : _*))

println(a)//DenseVector(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20)

//指定位置

println(a(0)) //1

//向量子集

println( a(1 to 4) )//DenseVector(2, 3, 4, 5)

println( a(1 until 4) )//DenseVector(2, 3, 4)

//指定开始位置至结尾

println( a(1 to -1) )//DenseVector(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20)

//按照指定步长去子集这个是倒序方式

println( a(5 to 0 by -1) )//DenseVector(6, 5, 4, 3, 2, 1)

//最后一个元素

println( a(-1)) //20

val m = DenseMatrix((1.0, 2.0, 3.0), (4.0, 5.0, 6.0))

println(m)

1.0 2.0 3.0

4.0 5.0 6.0

//指定位置

println( m(0 ,1) ) //2.0

//矩阵指定列

println( m(:: ,1) ) // DenseVector(2.0, 5.0)

//Breeze元素操作

//调整矩阵形状

val justAdjustMatrix =m.reshape(3, 2)

println(justAdjustMatrix)

DenseVector(2.0, 5.0)

1.0 5.0

4.0 3.0

2.0 6.0

//矩阵转成向量

val toVector=m.toDenseVector

println(toVector)//DenseVector(1.0, 4.0, 2.0, 5.0, 3.0, 6.0)

println(toVector.toDenseMatrix)//1.0 4.0 2.0 5.0 3.0 6.0

//复制下三角

println(lowerTriangular(m))//

1.0 0.0

4.0 5.0

//复制上三角

println(upperTriangular(m))

1.0 2.0

0.0 5.0

//矩阵复制

println(m.copy)

//m 原始为这样的

// 1.0 2.0

// 0.0 5.0

//取对角线元素

println(diag(upperTriangular(m)))

//DenseVector(1.0, 5.0) 个人觉得很怪异，不是应该为(1.0, 6.0)吗？上面上、下三角也好像出乎的意料

//子集赋数值

println(a(1 to 4 ):=5)

//(1.0, 5.0)

//子集赋向量

println( a(1 to 4):=DenseVector(1,2,3,4) )

//DenseVector(1, 2, 3, 4)

println(m)

//矩阵赋值

// println( m( 1 to 2, 1 to 2) := 0.0 )

//Exception in thread "main" java.lang.IndexOutOfBoundsException: Row slice of Range(1, 2) was bigger than matrix rows of 2

println("-==========m1================-")

println( m( 0 to 1, 1 to 2) := 0.0 )

println("-==========m================-")

println(m)

println("-==========m end================-")

-==========m1================-

0.0 0.0

-==========m================-

0.0 0.0 3.0

0.0 0.0 6.0

-==========m end================-

//矩阵列赋值

val re=m(::, 2) := 5.0

println(re.toDenseMatrix)

//5.0 5.0

val a1 = DenseMatrix((1.0, 2.0, 3.0), (4.0, 5.0, 6.0))

val a2 = DenseMatrix((7.0, 8.0, 9.0), (10.0, 11.0, 12.0))

//垂直连接矩阵

val verticalLike=DenseMatrix.vertcat(a1, a2)

println(verticalLike)

println("-==========================-")

1.0 2.0 3.0

4.0 5.0 6.0

7.0 8.0 9.0

10.0 11.0 12.0

//横向连接矩阵

val twoMatrixConn=DenseMatrix.horzcat( a1, a2)

println(twoMatrixConn)

println("-==========================-")

1.0 2.0 3.0 7.0 8.0 9.0

4.0 5.0 6.0 10.0 11.0 12.0

//向量的连接

val connnectVector1=DenseVector.vertcat(DenseVector(20, 21, 22), DenseVector(23, 24, 25))

val connnectVector2=DenseVector.horzcat(DenseVector(20, 21, 22), DenseVector(23, 24, 25))

println(connnectVector1)//DenseVector(20, 21, 22, 23, 24, 25)

println(connnectVector2)

20 23

21 24

22 25

//Breeze数值计算函数

//元素加法

println(a1 + a2)

8.0 10.0 12.0

14.0 16.0 18.0

//元素乘法

println(a1 :* a2)

7.0 16.0 27.0

40.0 55.0 72.0

//元素除法

println(a1 :/ a2)

0.14285714285714285 0.25 0.3333333333333333

0.4 0.45454545454545453 0.5

//元素比较

println(a1 :< a2)

true true true

//元素相等

println(a1 :== a2)

false false false

//元素追加

println(a1 :+=2.0)

3.0 4.0 5.0

6.0 7.0 8.0

//元素追乘

println(a1 :*=2.0)

6.0 8.0 10.0

12.0 14.0 16.0

//向量点积

val vectorDot=DenseVector(1, 2, 3, 4) dot DenseVector(1, 1, 1, 1)

println(vectorDot)//10

//元素最大值

println(max(a1))//16.0

//元素最小值

println(min(a1))//6.0

//元素最大值的位置

println(argmax(a1))// (1,2)

//元素最小值的位置

println(argmin(a1))// (0,0)

//Breeze求和函数

val m1 = DenseMatrix((1.0, 2.0, 3.0, 4.0), (5.0, 6.0, 7.0, 8.0), (9.0, 10.0, 11.0, 12.0))

println(m1)

1.0 2.0 3.0 4.0

5.0 6.0 7.0 8.0

9.0 10.0 11.0 12.0

println("-==========================-")

//元素求和

println(sum(m1))//78.0

//每一列求和

println(sum(m1, Axis._0))//res59: breeze.linalg.DenseMatrix[Double] = 15.0 18.0 21.0 24.0

//每一行求和

println(sum(m1, Axis._1))//res60: breeze.linalg.DenseVector[Double] = DenseVector(10.0, 26.0, 42.0)

//对角线元素和

println(trace(lowerTriangular(m1)))// res61: Double = 18.0

//累积和

val a3 = new DenseVector[Int](Array(10 to 20: _*))

println(accumulate(a3)) // DenseVector(10, 21, 33, 46, 60, 75, 91, 108, 126, 145, 165)

//Breeze布尔函数

val c = DenseVector(true, false, true)

val d = DenseVector(false, true, true)

//元素与操作

println(c :& d) // DenseVector(false, false, true)

//元素或操作

println(c :| d) //DenseVector(true, true, true)

//元素非操作

println(!c) //DenseVector(false, true, false)

val e = DenseVector[Int](-3, 0, 2)

//存在非零元素

println(any(e)) //true

//所有元素非零

println(all(e)) //false

//Breeze线性代数函数

val f = DenseMatrix((1.0, 2.0, 3.0), (4.0, 5.0, 6.0), (7.0, 8.0, 9.0))

val g = DenseMatrix((1.0, 1.0, 1.0), (1.0, 1.0, 1.0), (1.0, 1.0, 1.0))

//线性求解，AX = B，求解X

println(f \ g)

/* breeze.linalg.DenseMatrix[Double] =

-2.5 -2.5 -2.5

4.0 4.0 4.0

-1.5 -1.5 -1.5

//转置

println(f.t)

/* breeze.linalg.DenseMatrix[Double] =

1.0 4.0 7.0

2.0 5.0 8.0

3.0 6.0 9.0

//求特征值

println(det(f)) // Double = 6.661338147750939E-16

//求逆

println(inv(f))

-4.503599627370499E15 9.007199254740992E15 -4.503599627370495E15

9.007199254740998E15 -1.8014398509481984E16 9.007199254740991E15

-4.503599627370498E15 9.007199254740992E15 -4.5035996273704955E15

//求伪逆

println(pinv(f))

-3.7720834019330525E14 7.544166803866101E14 -3.77208340193305E14

7.544166803866094E14 -1.5088333607732208E15 7.544166803866108E14

-3.772083401933041E14 7.544166803866104E14 -3.772083401933055E14

//特征值和特征向量

println(eig(f))

Eig(DenseVector(16.116843969807043, -1.1168439698070427, -1.3036777264747022E-15),DenseVector(0.0, 0.0, 0.0),-0.23197068724628617 -0.7858302387420671 0.40824829046386363

-0.5253220933012336 -0.08675133925662833 -0.816496580927726

-0.8186734993561815 0.61232756022881 0.4082482904638625

)

//奇异值分解

val svd.SVD(u,s,v) = svd(g)

println(u)

-0.5773502691896255 -0.5773502691896257 -0.5773502691896256

-0.5773502691896256 -0.2113248654051871 0.7886751345948126

-0.5773502691896256 0.7886751345948129 -0.21132486540518708

println("==============================")

println(s) //DenseVector(3.0000000000000004, 0.0, 0.0)

println("==============================")

println(v)

-0.5773502691896256 -0.5773502691896257 -0.5773502691896256

0.0 -0.7071067811865474 0.7071067811865477

0.816496580927726 -0.4082482904638629 -0.4082482904638628

//求矩阵的秩

println(rank(f)) //2

//矩阵长度

println(f.size) //9

//矩阵行数

println(f.rows) // 3

//矩阵列数

f.cols // 3

//Breeze取整函数

val h = DenseVector(-1.2, 0.7, 2.3) // breeze.linalg.DenseVector[Double] = DenseVector(-1.2, 0.7, 2.3)

//四舍五入

println( round(h) ) // breeze.linalg.DenseVector[Long] = DenseVector(-1, 1, 2)

//大于它的最小整数

println( ceil(h) ) // breeze.linalg.DenseVector[Double] = DenseVector(-1.0, 1.0, 3.0)

//小于它的最大整数

println( floor(h) ) // breeze.linalg.DenseVector[Double] = DenseVector(-2.0, 0.0, 2.0)

//符号函数

println( signum(h) ) // breeze.linalg.DenseVector[Double] = DenseVector(-1.0, 1.0, 1.0)

//取正数

println( abs(h) ) // breeze.linalg.DenseVector[Double] = DenseVector(1.2, 0.7, 2.3)

}

spark 矩阵转置 java_Spark MLlib之使用Breeze操作矩阵向量

相关阅读

相关文章

相关问答

相关文档