Spark Tricks

# https://towardsdatascience.com/beginners-guide-to-pyspark-bbe3b553b79f

# 增加列
data = data.withColumn('date', data.data)

# 删除列
data = data.drop('data_changed')

# 重命名列
data = data.withColumnRenamed('date', 'data_changed')

# 查找列
data.select(['open', 'close', 'adjusted'])

# https://zhuanlan.zhihu.com/p/34901558
# https://xinancsd.github.io/Python/pyspark_save_hive_table.html

# https://stackoverflow.com/questions/30244910/how-to-pivot-spark-dataframe

# https://stackoverflow.com/questions/58404845/confusion-matrix-to-get-precsion-recall-f1score

# https://xinancsd.github.io/Python/spark-submit_usage.html

# 带 py 参数提交：核心是 py 脚本放最后，不带任何指示性参数（如：--py-files），则后续参数可传递给 py。
# https://stackoverflow.com/questions/32217160/can-i-add-arguments-to-python-code-when-i-submit-spark-job
spark-submit --master somemasterurl job.py --ngrams 3

# https://conda.github.io/conda-pack/spark.html
# note: script.py 中需要完整的 spark session 初始化

PYSPARK_PYTHON=./environment/bin/python \
spark-submit \
--conf spark.yarn.appMasterEnv.PYSPARK_PYTHON=./environment/bin/python \
--master yarn \
--deploy-mode cluster \
--archives environment.tar.gz#environment \
script.py

# https://queirozf.com/entries/jupyter-kernels-how-to-add-change-remove#change-kernel-name
conda create --name pyspark python=3.7 xgboost scikit-learn jupyter pandas py4j findspark
conda activate pyspark
ipython kernel install --name "pyspark" --user

#remove if config failed
conda env remove --name pyspark
jupyter kernelspec remove pyspark

jupyter kernelspec list
vi **/kernel.json

{
 "argv": [
  "/your_path/add_bin_to_path.sh",
  "/your_path/envs/pyspark/bin/python",
  "-m",
  "ipykernel_launcher",
  "-f",
  "{connection_file}"
 ],
 "display_name": "pyspark",
 "language": "python",
 "metadata": {
  "debugger": true
 },
 "env": {
   "MY_ENV_BIN_DIR": "/your_path/java/jdk/bin",
   "JAVA_HOME": "/your_path/java/jdk/",
   "SPARK_HOME": "/your_path/spark/",
   "PYSPARK_SUBMIT_ARGS": "--master yarn --deploy-mode client pyspark-shell"
 }
}

# add_bin_to_path.sh
#!/bin/bash
export PATH=${MY_ENV_BIN_DIR}:${PATH}
$@

# https://stackoverflow.com/questions/32742004/create-spark-dataframe-can-not-infer-schema-for-type
from pyspark.sql.types import FloatType
df = spark.createDataFrame([1.0, 2.0, 3.0], FloatType())
df.show()