Mac mojave 10.14.3开发环境配置
anaconda3: python3.6
Anaconda3-5.1.0-MacOSX-x86_64.pkg
pycharm:
pycharm-professional-2018.1.6.dmg
xgboost:
1 fanladdar
2 homebrew
/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
切换中国的源
cd "$(brew --repo)"
git remote set-url origin https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/brew.git
cd "$(brew --repo)/Library/Taps/homebrew/homebrew-core"
git remote set-url origin https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/homebrew-core.git
brew update
echo 'export HOMEBREW_BOTTLE_DOMAIN=https://mirrors.tuna.tsinghua.edu.cn/homebrew-bottles' >> ~/.bash_profile
source ~/.bash_profile
如果你想重置
cd "$(brew --repo)"
git remote set-url origin https://github.com/Homebrew/brew.git
cd "$(brew --repo)/Library/Taps/homebrew/homebrew-core"
git remote set-url origin https://github.com/Homebrew/homebrew-core.git
配置vim
brew install macvim
ln -s /usr/local/Cellar/macvim/版本号/MacVim.app/Contents/bin/vim /usr/local/bin/vim
ln -s /usr/local/Cellar/macvim/版本号/MacVim.app/Contents/bin/mvim /usr/local/bin/mvim
编辑~/.bash_profile让vim和mvim是一个东西
alias vi=vim
alias vim=mvim
生效
source ~/.bash_profile
vim ~/.vimrc
写入
set enc=utf-8
set nu "显示行号"
set showtabline=0 "隐藏顶部标签栏"
set guioptions-=r "隐藏右侧滚动条"
set guioptions-=L "隐藏左侧滚动条"
set guioptions-=b "隐藏底部滚动条"
set cursorline "突出显示当前行"
set cursorcolumn "突出显示当前列"
set langmenu=zh_CN.UTF-8 "显示中文菜单"
set nowrap "设置代码不折行"
set fileformat=unix "设置以unix的格式保存文件"
set cindent "设置C样式的缩进格式"
set tabstop=4 "一个 tab 显示出来是多少个空格,默认 8"
set shiftwidth=4 "每一级缩进是多少个空格"
set backspace+=indent,eol,start "set backspace&可以对其重置"
set showmatch "显示匹配的括号"
set scrolloff=5 "距离顶部和底部5行"
set laststatus=2 "命令行为两行"
set mouse=a "启用鼠标"
set selection=exclusive
set selectmode=mouse,key
set matchtime=5
set ignorecase "忽略大小写"
set incsearch
set hlsearch "高亮搜索项"
set noexpandtab "不允许扩展table"
set autoread
再创建
vim ~/.gvimrc
内容同上
3 编译环境
brew install cmake
brew install gcc6
4 编译xgboost
cd ~
git clone --recursive https://github.com/dmlc/xgboost
cd xgboost
cp make/config.mk ./config.mk
vim config.mk
修改
export CC = /usr/local/Cellar/gcc@6/6.5.0_2/bin/gcc-6
export CXX = /usr/local/Cellar/gcc@6/6.5.0_2/bin/g+±6
保存退出
make -j4
cd python-package
python3 setup.py install
import xgboost
5 其他开发环境
java8
首先到oracle官网下载jdk8的mac包
http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html
我下载的是jdk-8u221-macosx-x64.dmg
双击安装后打开Finder, 在菜单栏–>前往–>文件夹–> /Library可以看到java路径
或者终端输入/usr/libexec/java_home 可以得到JAVA_HOME 的路径
vim ~/.bash_profile
export JAVA_HOME="/Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home"
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH
source ~/.bash_profile
终端输入java -version可以查看效果
安装scala
brew update
brew install scala@2.12
会自动给你装到/usr/local下面,就不用自己解压缩什么的了
vim ~/.bash_profile
export SCALA_HOME="/usr/local/Cellar/scala/2.12.8"
export PATH=${SCALA_HOME}/bin:$PATH
source ~/.bash_profile
终端输入scala -version可以看效果
安装spark
http://spark.apache.org/downloads.html
我下载的是spark-2.4.3-bin-hadoop2.7.tgz
spark documentation指明了2.4.3配套scala2.12.x
cd ~/Downloads
tar -zxvf spark-2.3.0-bin-hadoop2.7.tgz #得到spark-2.3.0-bin-hadoop2.7
sudo mv ~/Downloads/spark-2.4.3-bin-hadoop2.7 /usr/local/Cellar
vim ~/.bash_profile
export SPARK_HOME=/usr/local/Cellar/spark-2.3.0-bin-hadoop2.7
export PATH=${SPARK_HOME}/bin:$PATH
export PYTHONPATH=${SPARK_HOME}/python
source ~/.bash_profile
pip install py4j
终端输入spark-shell进入scala界面的交互式spark
终端输入pyspark进入python界面的交互式spark
其实如果再在bash_profile里面加上这样两句的话,终端输入pyspark会直接用jupyter来打开
export PYSPARK_DRIVER_PYTHON=ipython
export PYSPARK_DRIVER_PYTHON_OPTS="notebook"
以上创建的交互式环境都是不需要建立上下文的!!
记得把pyspark_home下面的python下面的pyspark目录整个拷贝到anaconda3/lib/python3.6/site_packages下面,这样pycharm才可以用
mac下面可以装f.lux来保护视力
确保mac定位服务开启
brew services是用来管理开机启动的
brew services list # 查看使用brew安装的服务列表
brew services run formula|–all # 启动服务(仅启动不注册)
brew services start formula|–all # 启动服务,并注册
brew services stop formula|–all # 停止服务,并取消注册
brew services restart formula|–all # 重启服务,并注册
brew services cleanup # 清除已卸载应用的无用的配置
安装maven
https://link.jianshu.com/?t=https://maven.apache.org/download.cgi
下载最新版本并解压
vim ~/.bash_profile
# maven environment
MAVEN_HOME=/Users/用户名/apache-maven-3.6.1
PATH=$MAVEN_HOME/bin:$PATH
export MAVEN_HOME
export PATH
最后用source ~/.bash_profile生效
vim ~/apache-maven-3.6.1/conf/setting.xml
在mirror部分添加
<mirrors>
<mirror>
<id>alimaven</id>
<name>aliyun maven</name>
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
<mirrorOf>central</mirrorOf>
</mirror>
</mirrors>
在localrepository添加
<localRepository>/Users/用户名/maven_repository/repository</localRepository>
idea的使用:
(mac里面用command+shift+G可以打开路径输入框,这样就可以访问根目录了)
configure->structure for new projects->在弹出的界面依次选择Project-New-JDK,选择JAVA_HOME,此时platform_settings 的sdks下面就会多出一项
configure->structure for new projects->platform settings->在Global Libraries选项卡下点+后选择Scala SDK,然后会弹出一个界面,选择scala_home/idea/lib
configure->structure for new projects->platform settings->在Global Libraries选项卡下点+后选择java,然后会弹出一个界面,将SPARK_HOME/jars目录下所有jar包全选。这个jar包可以重命名为spark_jar
new>Project->scala->会创建一个自定义名字的工程,里面含有一个叫的src文件夹默认为源,在这个源文件夹上右键New->Scala Class, Kind选择“Object”,编辑内容如下:
package main.scala
import org.apache.spark.{SparkConf, SparkContext}
object ss {
// 假设工程文件夹里有一个user.txt表
def main(args: Array[String]): Unit = {
val logFile = "users.txt"
val conf = new SparkConf().setAppName("ss").setMaster("local")
val sc = new SparkContext(conf)
val rdd = sc.textFile(logFile)
rdd.foreach(println)
sc.stop()
}
}