notebook20180415

最新推荐文章于 2021-01-15 16:31:52 发布

junamistudio

最新推荐文章于 2021-01-15 16:31:52 发布

阅读量291

点赞数

分类专栏： Python 文章标签： notebook

本文链接：https://blog.csdn.net/junami/article/details/81839360

版权

Python 专栏收录该内容

2 篇文章 0 订阅

订阅专栏

curl

curl -F "userid=1" -F "filecomment=This is an image file" -F "image=@/home/user1/Desktop/test.jpg" localhost/uploader.php
curl -i -F "filename=@样例.xlsx;type=application/octet-stream" http://127.0.0.1:5001/upload_overdue_file
curl --request POST --data-binary "c:/样例.xlsx" --header "application/octet-stream" http://127.0.0.1:5001/upload_overdue_file
multipart/form-data
curl -i -X POST -H "Content-Type: multipart/form-data" -F "file=c:/样例.xlsx" http://127.0.0.1:5001/upload_overdue_file
curl -i -X POST -H "Content-Type: multipart/form-data" -F "file=c:/样例.xlsx" http://127.0.0.1:5001/upload_overdue_file
curl -F "file=c:/样例.xlsx" -F "yourname=Daniel" -F "filedescription=Cool text file with cool text inside" http://127.0.0.1:5001/upload_file
curl -v -H "Content-Type: application/x-www-form-urlencoded" -F "file=@c:/样例.xlsx" http://127.0.0.1:5001/upload_file
#FINAL, CHANGE FILE permission
curl -v  -F "file=@c:/test.xlsx"   http://127.0.0.1:5001/upload_file

free

free -g && sync && echo 3 > /proc/sys/vm/drop_caches && free -g

Upgrade Mysql 5.6 to 5.7

https://dev.mysql.com/doc/refman/5.7/en/upgrading.html#upgrade-methods
1. backup
mysqldump -u root -p --add-drop-table --routines --events --all-databases --force > data-for-upgrade.sql
2. shutdown
mysql -u root -p --execute="SET GLOBAL innodb_fast_shutdown=0"
mysqladmin -u root -p shutdown
sudo service mysqld status
3. upgrade binary with yum
wget https://repo.mysql.com//mysql57-community-release-el7-11.noarch.rpm
sudo yum localinstall mysql57-community-release-el7-11.noarch.rpm
4. mysqld_safe --user=mysql --datadir=/mnt/mysql
5. mysql_upgrade -u root -p

wget https://repo.mysql.com//mysql57-community-release-el7-11.noarch.rpm
sudo yum localinstall mysql57-community-release-el7-11.noarch.rpm -y
sudo yum install mysql-community-client.x86_64 -y
sudo service mysqld start

Url decode on Mysql

https://blog.cles.jp/item/4564
http://psoug.org/snippet/mySQL-url_decode-that-respects-utf8-characters-_502.htm

DELIMITER $$

DROP FUNCTION IF EXISTS `url_decode` $$
CREATE DEFINER=`root`@`%` FUNCTION `url_decode`(original_text TEXT) RETURNS TEXT CHARSET utf8
BEGIN
     DECLARE new_text TEXT DEFAULT NULL;
     DECLARE pointer INT DEFAULT 1;
     DECLARE end_pointer INT DEFAULT 1;
     DECLARE encoded_text TEXT DEFAULT NULL;
     DECLARE result_text TEXT DEFAULT NULL;

     SET new_text = REPLACE(original_text,'+',' ');

     SET pointer = LOCATE("%", new_text);
     while pointer <> 0 && pointer < (CHAR_LENGTH(new_text) - 2) DO
          SET end_pointer = pointer + 3;
          while MID(new_text, end_pointer, 1) = "%" DO
               SET end_pointer = end_pointer+3;
          END while;

          SET encoded_text = MID(new_text, pointer, end_pointer - pointer);
          SET result_text = CONVERT(UNHEX(REPLACE(encoded_text, "%", "")) USING utf8);
          SET new_text = REPLACE(new_text, encoded_text, result_text);
          SET pointer = LOCATE("%", new_text, pointer + CHAR_LENGTH(result_text));
     END while;

     return new_text;

END $$

DELIMITER ;

Nginx+Gunicorn+Flask

1. sudo apt-get install nginx

# site config
/etc/nginx/sites-available/mysite
--------
upstream my_servers {
  server unix:/home/hello/gunicorn.sock fail_timeout=0;
}

server {

  server_tokens off;

  listen 5005 ssl;
  ssl on;
  ssl_certificate /home/hello/ca/server.crt;
  ssl_certificate_key /home/hello/ca/server.key;
  ssl_protocols        SSLv3 TLSv1 TLSv1.1 TLSv1.2;
  ssl_ciphers          HIGH:!aNULL:!MD5;
  ssl_prefer_server_ciphers on;
  access_log /var/log/nginx/my.access.log;

  gzip on;
  gzip_types *;
  gzip_proxied any;

  location / {
    proxy_set_header Host $http_host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    proxy_pass       http://my_servers;
  }
}

http {
    log_format upstream_time '$remote_addr - $remote_user [$time_local] '
                             '"$request" $status $body_bytes_sent '
                             '"$http_referer" "$http_user_agent"'
                             'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';

    server {
        access_log /spool/logs/nginx-access.log upstream_time;
        ...
    }
}

--------
sudo ln -s /etc/nginx/sites-available/mysite /etc/nginx/sites-enabled/mysite
sudo /etc/init.d/nginx restart

2. Gunicorn
gunicorn -b unix:/home/hello/gunicorn.sock --name appname -w 4 --max-requests 1000 wsgi:app

CentOS 7 Nginx

sudo rpm -Uvh http://nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpm
sudo yum install -y nginx
mkdir /etc/nginx/sites-available
mkdir /etc/nginx/sites-enabled


sudo systemctl enable nginx
sudo systemctl start nginx
sudo systemctl enable nginx
sudo systemctl status nginx

### Python 3.5 ###
mkdir /usr/python3.5
./configure --prefix=/usr/python3.5
make
make install



/usr/python3.5/bin/pip3.5 install virtualenv
 virtualenv --no-site-packages venv_py35 -p /usr/python3.5/bin/python3.5
/usr/python3.5/bin/virtualenv --no-site-packages venv_py35 -p /usr/python3.5/bin/python3.5
pip install flask schedule flask_httpauth pyDes pymysql
pip install future


### Launch Script ####
#!/bin/bash
PY_VENV='/mnt/home/dingjunnan/venv_py35'
NAME='credit_api'
HOME='/mnt/home/dingjunnan'
PROJECT_DIR=${HOME}/credit_api
SOCKFILE=${HOME}/sock/credit_gunicorn.sock
WORKERS=4
echo "starting $NAME as `whoami`"

source ${PY_VENV}/bin/activate
cd ${PROJECT_DIR}


gunicorn -b unix:${SOCKFILE} --name ${NAME} -w ${WORKERS} --max-requests 1000 --log-level=info wsgi:app

cloudera + CentOS7


# disable SELINUX
$ vi /etc/selinux/config
SELINUX=disabled

# disable firewalld
systemctl stop firewalld
systemctl disable firewalld
systemctl status firewalld


# User & Group
sudo groupadd cloudera-scm
sudo useradd --system --home=/opt/cm-5.12.1/run/cloudera-scm-server --shell=/bin/false --comment "Cloudera SCM User" -g cloudera-scm cloudera-scm

# chmod
sudo chown -R cloudera-scm:cloudera-scm /opt/cloudera
sudo chown -R cloudera-scm:cloudera-scm /opt/cm-5.12.1

# make dir
$sudo mkdir /var/lib/cloudera-scm-server
$sudo chown cloudera-scm:cloudera-scm /var/lib/cloudera-scm-server
$sudo mkdir /var/log/cloudera-scm-server
$sudo chown cloudera-scm:cloudera-scm /var/log/cloudera-scm-server

# hostname
$ sudo vi /etc/sysconfig/network
HOSTNAME=bigdatadev
$ sudo vi /etc/hosts
10.5.202.52 bigdatadev
$ hostname bigdatadev
$ /etc/init.d/network restart

# mysql
select @@validate_password_policy;
set global validate_password_policy=0;
set global validate_password_length=4;



ALTER USER 'root'@'localhost' IDENTIFIED BY 'Hello@123';
GRANT ALL PRIVILEGES ON *.* TO root@"%" IDENTIFIED BY "Hello@123";
flush privileges;
GRANT ALL ON *.* TO 'root'@'%' IDENTIFIED BY "Hello@123";
FLUSH PRIVILEGES;
SHOW GRANTS FOR 'bigdata'@'%';

create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database reports_mgr DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
GRANT ALL PRIVILEGES ON *.* TO root@"%";

sysctl -w net.ipv6.conf.all.disable_ipv6=1
sysctl -w net.ipv6.conf.default.disable_ipv6=1


./scm_prepare_database.sh mysql cm -hlocalhost -uroot -pHello@123 bigdatadev scm scm scm

Hive on spark configure

spark.executor.cores    Between 5-7, See tuning details section
spark.executor.memory   yarn.nodemanager.resource.memory-mb * (spark.executor.cores / yarn.nodemanager.resource.cpu-vcores) 
30 * (5/40)   
spark.yarn.executor.memoryOverhead  15-20% of spark.executor.memory
spark.executor.instances    Depends on spark.executor.memory + spark.yarn.executor.memoryOverhead, see tuning details section.

set hive.execution.engine=spark;
set spark.executor.cores=5;
set spark.executor.memory=10GB;
set spark.yarn.executor.memoryOverhead=3GB;
set spark.executor.instances=20GB;

set hive.execution.engine=spark;
set spark.executor.cores=4;
set spark.executor.memory=13GB;
set spark.yarn.executor.memoryOverhead=2GB;
set spark.executor.instances=15GB;

PYSPARK


#Horton .bashrc
export PYTHONPATH=${PYTHONPATH}:/usr/hdp/current/spark-client/python:/usr/hdp/current/spark-client/python/lib/py4j-0.9-src.zip
export SPARK_HOME=/usr/hdp/current/spark-client

#Cloudera .bashrc
export PYTHONPATH=${PYTHONPATH}:/opt/cloudera/parcels/SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354/lib/spark2/python:/opt/cloudera/parcels/SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354/lib/spark2/python/lib/py4j-0.10.4-src.zip
export SPARK_HOME=/opt/cloudera/parcels/SPARK2-2.2.0.cloudera1-1.cdh5.12.0.p0.142354/lib/spark2

#Python packages
pip install findspark

vbox

wmic diskdrive list
wmic diskdrive list brief 
VBoxManage internalcommands listpartitions -rawdisk \\.\PhysicalDrive1
C:\Program Files\Oracle\VirtualBox>VBoxManage.exe internalcommands createrawvmdk  -filename F:\VirtualBoxRealDisk\PHYSICALDRIVE2.vmdk -rawdisk \\.\PHYSICALDRIVE2
C:\Program Files\Oracle\VirtualBox>VBoxManage.exe internalcommands createrawvmdk  -filename C:\VirtualBoxRealDisk\PHYSICALDRIVE3P3.vmdk -rawdisk \\.\PHYSICALDRIVE3 -partitions 3

superset

http://blog.csdn.net/dzjun/article/details/62421718
https://superset.incubator.apache.org/installation.html
https://github.com/apache/incubator-superset

Redash

https://blog.dshimizu.jp/article/225

command = “mysqldump -u%s -p%s –opt -R -d %s -h%s > %s” % (
server_user_name, server_user_passwd, server_db_schema, server_ip, des_path)
command = “mysqldump -u%s -p%s -t %s -h%s –triggers=false > %s” % (
server_user_name, server_user_passwd, server_db_schema, server_ip, des_path)

junamistudio

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
notebook20180415

curlcurl -F &quot;userid=1&quot; -F &quot;filecomment=This is an image file&quot; -F &quot;image=@/home/user1/Desktop/test.jpg&quot; localhost/uploader.phpcurl -i -F &quot;filename=@样例.xlsx;type=application/octet-stream&amp
复制链接

扫一扫