Skip to content
Merged
2 changes: 2 additions & 0 deletions .github/workflows/scala-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ jobs:
fail-fast: false
matrix:
include:
- spark-version: 3.5.7
java-version: 8
- spark-version: 3.5.7
java-version: 11
Comment thread
james-willis marked this conversation as resolved.
- spark-version: 3.5.7
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import org.openjdk.jmh.annotations.*

import java.io.File
import java.nio.file.Path
import java.nio.file.Paths

trait LDBCBenchmarkBase {
@Param(Array("wiki-Talk"))
Expand All @@ -19,7 +20,7 @@ trait LDBCBenchmarkBase {
var spark: SparkSession = _
var graph: GraphFrame = _

protected def cacheDir: Path = Path.of(new File("target").toURI).resolve("ldbc-cache")
protected def cacheDir: Path = Paths.get(new File("target").toURI).resolve("ldbc-cache")

@Setup(Level.Trial)
def setup(): Unit = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import org.openjdk.jmh.infra.Blackhole
import java.io.File
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.util.Properties
import java.util.concurrent.TimeUnit

Expand Down Expand Up @@ -85,7 +86,7 @@ class LDBCBenchmarkSuite {

private def caseRoot: Path = resourcesPath.resolve(benchmarkGraphName)

private def resourcesPath = Path.of(new File("target").toURI)
private def resourcesPath = Paths.get(new File("target").toURI)

@Benchmark
def benchmarkSP(blackhole: Blackhole): Unit = {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
package org.graphframes.benchmarks

import java.net.URL
import java.nio.file.Files
import java.nio.file.Path
import scala.sys.process.*

class ParquetDataLoader(cacheDir: Path) {
private val LDBC_PARQUET_URL_PREFIX = "https://datasets.ldbcouncil.org/graphalytics-parquet/"
private val bufferSize = 8192

def downloadParquetIfNotExists(graphName: String): Unit = {
val graphDir = cacheDir.resolve(graphName)
Expand All @@ -33,21 +32,31 @@ class ParquetDataLoader(cacheDir: Path) {
}
}

// Use curl instead of Java's URLConnection because the LDBC CDN (Cloudflare)
Comment thread
james-willis marked this conversation as resolved.
// rejects Java 8's TLS fingerprint with HTTP 403.
// TODO: restore URLConnection after Spark 3.5.x EOL (~April 2026) when JDK 8 can be dropped:
// private def downloadFile(url: String, dest: Path): Unit = {
// val connection = new java.net.URL(url).openConnection()
// connection.setConnectTimeout(30000)
// connection.setReadTimeout(30000)
// val inputStream = connection.getInputStream
// val outputStream = Files.newOutputStream(dest)
// val buffer = new Array[Byte](8192)
// var bytesRead = 0
// try {
// while ({ bytesRead = inputStream.read(buffer); bytesRead } != -1) {
// outputStream.write(buffer, 0, bytesRead)
// }
// } finally {
// inputStream.close()
// outputStream.close()
// }
// println(s"Downloaded $url to $dest")
// }
private def downloadFile(url: String, dest: Path): Unit = {
val connection = new URL(url).openConnection()
connection.setConnectTimeout(30000)
connection.setReadTimeout(30000)
val inputStream = connection.getInputStream
val outputStream = Files.newOutputStream(dest)
val buffer = new Array[Byte](bufferSize)
var bytesRead = 0
try {
while ({ bytesRead = inputStream.read(buffer); bytesRead } != -1) {
outputStream.write(buffer, 0, bytesRead)
}
} finally {
inputStream.close()
outputStream.close()
val curlExit = s"curl -fSL -o ${dest.toString} $url".!
if (curlExit != 0) {
throw new RuntimeException(s"Failed to download $url (curl exit code: $curlExit)")
}
println(s"Downloaded $url to $dest")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ object KMinSampling extends Serializable {
// That is very stupid way actually. But it is the only way with public API
spark
.createDataFrame(
java.util.List.of[Row](),
java.util.Collections.emptyList[Row](),
StructType(
StructField(colNames(0), dataType) :: StructField(colNames(1), LongType) :: Nil))
.encoder
Expand Down
30 changes: 18 additions & 12 deletions core/src/main/scala/org/graphframes/examples/LDBCUtils.scala
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
package org.graphframes.examples

import java.net.URL
import java.nio.file.*
import scala.sys.process.*

object LDBCUtils {
private val LDBC_URL_PREFIX = "https://datasets.ldbcouncil.org/graphalytics/"
private val bufferSize = 8192 // 8Kb

val TEST_BFS_DIRECTED = "test-bfs-directed"
val TEST_BFS_UNDIRECTED = "test-bfs-undirected"
Expand Down Expand Up @@ -51,7 +49,7 @@ object LDBCUtils {
CIT_PATENTS,
WIKI_TALKS)

private def ldbcURL(caseName: String): URL = new URL(s"${LDBC_URL_PREFIX}${caseName}.tar.zst")
private def ldbcURL(caseName: String): String = s"${LDBC_URL_PREFIX}${caseName}.tar.zst"

private def checkZSTD(): Unit = {
try {
Expand Down Expand Up @@ -81,16 +79,24 @@ object LDBCUtils {
Files.createDirectories(dir)
}
val archivePath = path.resolve(s"${name}.tar.zst")
val connection = ldbcURL(name).openConnection()
val inputStream = connection.getInputStream
val outputStream = Files.newOutputStream(archivePath)
val buffer = new Array[Byte](bufferSize)
var bytesRead = 0
while ({ bytesRead = inputStream.read(buffer); bytesRead } != -1) {
outputStream.write(buffer, 0, bytesRead)
// Use curl instead of Java's URLConnection because the LDBC CDN (Cloudflare)
// rejects Java 8's TLS fingerprint with HTTP 403.
// TODO: restore URLConnection after Spark 3.5.x EOL (~April 2026) when JDK 8 can be dropped:
// val connection = new java.net.URL(ldbcURL(name)).openConnection()
// val inputStream = connection.getInputStream
// val outputStream = Files.newOutputStream(archivePath)
// val buffer = new Array[Byte](8192)
// var bytesRead = 0
// while ({ bytesRead = inputStream.read(buffer); bytesRead } != -1) {
// outputStream.write(buffer, 0, bytesRead)
// }
// inputStream.close()
// outputStream.close()
val curlExit = s"curl -fSL -o ${archivePath.toString} ${ldbcURL(name)}".!
if (curlExit != 0) {
throw new RuntimeException(
s"Failed to download ${ldbcURL(name)} (curl exit code: $curlExit)")
}
inputStream.close()
outputStream.close()
println(s"Uncompressing ${archivePath.toString} to ${dir.toString}...")
s"zstd -d ${archivePath.toString} -o ${archivePath.toString.replace(".zst", "")}".!
s"tar -xf ${archivePath.toString.replace(".zst", "")} -C ${dir.toString}".!
Expand Down
Loading