homework4

Messaging in Docker images

我在这里使用了python的web服务作为生产者，go服务作为消费者。
两个服务的目录结构如下：

python-service

1. 使用Flask来构建，首先查看dockerfile

FROM debian:stretch-slim

# ensure local python is preferred over distribution python
ENV PATH /usr/local/bin:$PATH

# http://bugs.python.org/issue19846
# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
ENV LANG C.UTF-8

# runtime dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
		ca-certificates \
		libexpat1 \
		libffi6 \
		libgdbm3 \
		libreadline7 \
		libsqlite3-0 \
		libssl1.1 \
	&& rm -rf /var/lib/apt/lists/*

ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D
ENV PYTHON_VERSION 3.6.5

RUN set -ex \
	&& buildDeps=" \
		dpkg-dev \
		gcc \
		libbz2-dev \
		libc6-dev \
		libexpat1-dev \
		libffi-dev \
		libgdbm-dev \
		liblzma-dev \
		libncursesw5-dev \
		libreadline-dev \
		libsqlite3-dev \
		libssl-dev \
		make \
		tcl-dev \
		tk-dev \
		wget \
		xz-utils \
		zlib1g-dev \
# as of Stretch, "gpg" is no longer included by default
		$(command -v gpg > /dev/null || echo 'gnupg dirmngr') \
	" \
	&& apt-get update && apt-get install -y $buildDeps --no-install-recommends && rm -rf /var/lib/apt/lists/* \
	\
	&& wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
	&& wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
	&& export GNUPGHOME="$(mktemp -d)" \
	&& gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \
	&& gpg --batch --verify python.tar.xz.asc python.tar.xz \
	&& rm -rf "$GNUPGHOME" python.tar.xz.asc \
	&& mkdir -p /usr/src/python \
	&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \
	&& rm python.tar.xz \
	\
	&& cd /usr/src/python \
	&& gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \
	&& ./configure \
		--build="$gnuArch" \
		--enable-loadable-sqlite-extensions \
		--enable-shared \
		--with-system-expat \
		--with-system-ffi \
		--without-ensurepip \
	&& make -j "$(nproc)" \
	&& make install \
	&& ldconfig \
	\
	&& apt-get purge -y --auto-remove $buildDeps \
	\
	&& find /usr/local -depth \
		\( \
			\( -type d -a \( -name test -o -name tests \) \) \
			-o \
			\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
		\) -exec rm -rf '{}' + \
	&& rm -rf /usr/src/python

# make some useful symlinks that are expected to exist
RUN cd /usr/local/bin \
	&& ln -s idle3 idle \
	&& ln -s pydoc3 pydoc \
	&& ln -s python3 python \
	&& ln -s python3-config python-config

# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
ENV PYTHON_PIP_VERSION 9.0.3

RUN set -ex; \
	\
	apt-get update; \
	apt-get install -y --no-install-recommends wget; \
	rm -rf /var/lib/apt/lists/*; \
	\
	wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; \
	\
	apt-get purge -y --auto-remove wget; \
	\
	python get-pip.py \
		--disable-pip-version-check \
		--no-cache-dir \
		"pip==$PYTHON_PIP_VERSION" \
	; \
	pip --version; \
	\
	find /usr/local -depth \
		\( \
			\( -type d -a \( -name test -o -name tests \) \) \
			-o \
			\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
		\) -exec rm -rf '{}' +; \
	rm -f get-pip.py

RUN pip install pika

RUN pip install Flask

# Create and define the container's working directory.
RUN mkdir /python-service
WORKDIR /python-service

2. 创建main.py文件

from flask import Flask
from flask import request
from flask import jsonify
from services.user_event_handler import emit_user_profile_update

app = Flask(__name__)

@app.route('/users/<int:user_id>', methods=['POST'])
def update(user_id):
    new_name = request.form['full_name']

    # Update the user in the datastore using a local transaction...
    
    emit_user_profile_update(user_id, {'full_name': new_name})

    return jsonify({'full_name': new_name}), 201

3. 在service文件夹下创建user_event_handler.py文件以及一个空的__init__.py

import pika
import json

def emit_user_profile_update(user_id, new_data):
    # 'rabbitmq-server' is the network reference we have to the broker, 
    # thanks to Docker Compose.
    connection = pika.BlockingConnection(pika.ConnectionParameters(host='rabbitmq-server'))
    channel    = connection.channel()

    exchange_name = 'user_updates'
    routing_key   = 'user.profile.update'

    # This will create the exchange if it doesn't already exist.
    channel.exchange_declare(exchange=exchange_name, exchange_type='topic', durable=True)
    
    new_data['id'] = user_id

    channel.basic_publish(exchange=exchange_name,
                          routing_key=routing_key,
                          body=json.dumps(new_data),
                          # Delivery mode 2 makes the broker save the message to disk.
                          properties=pika.BasicProperties(
                            delivery_mode = 2,
                        ))

    print("%r sent to exchange %r with data: %r" % (routing_key, exchange_name, new_data))
    connection.close()

go-service

1. dockerfile:

  1 FROM golang:latest
  2 RUN go get github.com/streadway/amqp
  3 
  4 # Create and define the container's working directory.
  5 RUN mkdir /go-service
  6 WORKDIR /go-service

2. main.go:

package main

import (
	"fmt"
	"log"
	"github.com/streadway/amqp"
)

func failOnError(err error, msg string) {
	if err != nil {
		// Exit the program.
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func main() {
	// 'rabbitmq-server' is the network reference we have to the broker, 
	// thanks to Docker Compose.
	conn, err := amqp.Dial("amqp://guest:guest@rabbitmq-server:5672/")
	failOnError(err, "Error connecting to the broker")
	// Make sure we close the connection whenever the program is about to exit.
	defer conn.Close()

	ch, err := conn.Channel()
	failOnError(err, "Failed to open a channel")
	// Make sure we close the channel whenever the program is about to exit.
	defer ch.Close()
	
	exchangeName := "user_updates"
	bindingKey   := "user.profile.*"

	// Create the exchange if it doesn't already exist.
	err = ch.ExchangeDeclare(
			exchangeName, 	// name
			"topic",  		// type
			true,         	// durable
			false,
			false,
			false,
			nil,
	)
	failOnError(err, "Error creating the exchange")
	
	// Create the queue if it doesn't already exist.
	// This does not need to be done in the publisher because the
	// queue is only relevant to the consumer, which subscribes to it.
	// Like the exchange, let's make it durable (saved to disk) too.
	q, err := ch.QueueDeclare(
			"",    // name - empty means a random, unique name will be assigned
			true,  // durable
			false, // delete when unused
			false, 
			false, 
			nil,   
	)
	failOnError(err, "Error creating the queue")

	// Bind the queue to the exchange based on a string pattern (binding key).
	err = ch.QueueBind(
			q.Name,       // queue name
			bindingKey,   // binding key
			exchangeName, // exchange
			false,
			nil,
	)
	failOnError(err, "Error binding the queue")

	// Subscribe to the queue.
	msgs, err := ch.Consume(
			q.Name, // queue
			"",     // consumer id - empty means a random, unique id will be assigned
			false,  // auto acknowledgement of message delivery
			false,  
			false,  
			false,  
			nil,
	)
	failOnError(err, "Failed to register as a consumer")


	forever := make(chan bool)

	go func() {
		for d := range msgs {
			log.Printf("Received message: %s", d.Body)

			// Update the user data on the service's 
			// associated datastore using a local transaction...

			// The 'false' indicates the success of a single delivery, 'true' would mean that
			// this delivery and all prior unacknowledged deliveries on this channel will be
			// acknowledged, which I find no reason for in this example.
			d.Ack(false)
		}
	}()
	
	fmt.Println("Service listening for events...")
	
	// Block until 'forever' receives a value, which will never happen.
	<-forever
}

使用Docker Compose合并起来

  1 version: "3.2"
  2 services:
  3     rabbitmq-server:
  4         image: rabbitmq:latest
  5 
  6     python-service:
  7         build: ./python-service
  8         # 'rabbitmq-server' will be available as a network reference inside     this service 
  9         # and this service will start only after the RabbitMQ service has.
 10         depends_on:
 11             - rabbitmq-server
 12         # Keep it running.  
 13         tty: true
 14         # Map port 3000 on the host machine to port 3000 of the container.
 15         ports:
 16             - "3000:3000"
 17         volumes:
 18             - './python-service:/python-service'
 19 
 20     go-service:
 21         build: ./go-service
 22         depends_on:
 23             - rabbitmq-server
 24         tty: true
 25         volumes:
 26             - './go-service:/go-service'
 27 
 28 # Host volumes used to store code.
 29 volumes:
 30     python-service:
 31     go-service:

其中

• /go-service 为Go service 的容器工作目录
• /python-service 为Python service 的容器工作目录

1. 运行docker-compose up，并使用docker-compose ps查看运行状态
2. 正常运行后开启两个新终端，分为连接python与go

docker exec -it microservicesusingrabbitmq_python-service_1 bash
FLASK_APP=main.py python -m flask run — port 3000 — host 0.0.0.0

docker exec -it microservicesusingrabbitmq_go-service_1 bash
go run main.go

3. 使用curl -d “full_name=usama” -X POST http://localhost:3000/users/1发送消息
4. 接下来你会看到这些事件：
   
   
5. 同时在rabbitmq中可以看到日志信息：
   

测试

使用脚本循环串行测试

我使用如下脚本内容循环发送请求，同时生产者消费者都正常运行，消息可以正常传递：

  1 #/bin/sh
  2 sum=1000000000
  3 i=0
  4 while [ "$i" -le "$sum" ]
  5 do
  6 curl -d "full_name=$i" -X POST http://localhost:3000/users/1
  7 i=$(($i+1))
  8 done

关闭消费者测试

1. 关闭消费者，再次使用脚本运行
   
2. 关闭了自动消息应答，手动也未设置应答，这是一个很简单错误，但是后果却是极其严重的。消息在分发出去以后，得不到回应，所以不会在内存中删除，结果RabbitMQ会越来越占用内存，最终虚拟机崩了，直接卡死。