Search code examples
apache-apisix

Unable to load custom plugins in Apache Apisix


I followed the plugin develop documentation to create a new plugin and modify the existing authz-casbin plugin.

Here is what my folder tree looks like (I cant post picture):

.
├── all-in-one
│   └── apisix-dashboard
│       └── conf.yaml
├── apisix
│   └── plugins
│       └── custom-plugins
│           ├── authz-casbin.lua
│           └── jwt-extractor.lua
├── apisix_conf
│   ├── apisix-standalone.yaml
│   └── config.yaml
├── apisix-dashboard
│   └── conf.yaml
├── apisix-plugins
├── apisix_plugins
├── docker-compose-arm64.yml
├── docker-compose-standalone.yml
├── docker-compose.yml
├── etcd_conf
│   └── etcd.conf.yml
├── grafana_conf
│   ├── config
│   │   └── grafana.ini
│   ├── dashboards
│   │   └── apisix-grafana-dashboard.json
│   └── provisioning
│       ├── dashboards
│       │   └── all.yaml
│       └── datasources
│           └── all.yaml
├── mkcert
│   ├── lvh.me+1-key.pem
│   ├── lvh.me+1.pem
│   ├── README.md
│   ├── rootCA-key.pem
│   └── rootCA.pem
├── prometheus_conf
│   └── prometheus.yml
└── upstream
    ├── web1.conf
    └── web2.conf

This is my apisix conf.yaml file

apisix:
  node_listen: 9080              # APISIX listening port
  enable_ipv6: false
  # Add these lines
  extra_lua_path: "/usr/local/apisix/custom-plugin/?.lua"
  log_level: debug
  enable_control: true
  control:
    ip: "0.0.0.0"
    port: 9092
  

deployment:
  admin:
    allow_admin:               # https://nginx.org/en/docs/http/ngx_http_access_module.html#allow
      - 0.0.0.0/0              # We need to restrict ip access rules for security. 0.0.0.0/0 is for test.

    admin_key:
      - name: "admin"
        key: edd1c9f034335f136f87ad84b625c8f1
        role: admin                 # admin: manage all configuration data

      - name: "viewer"
        key: 4054f7cf07e344346cd3f287985e76a2
        role: viewer

  etcd:
    host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
      - "http://etcd:2379"          # multiple etcd address
    prefix: "/apisix"               # apisix configurations prefix
    timeout: 30                     # 30 seconds

plugin_attr:
  prometheus:
    export_addr:
      ip: "0.0.0.0"
      port: 9091

Here is my apisix dashboard conf.yaml

conf:
  listen:
    host: 0.0.0.0     # `manager api` listening ip or host name
    port: 9000          # `manager api` listening port
  etcd:
    endpoints:          # supports defining multiple etcd host addresses for an etcd cluster
      - etcd:2379

                        # etcd basic auth info
    # username: "root"    # ignore etcd username if not enable etcd auth
    # password: "123456"  # ignore etcd password if not enable etcd auth
  log:
    error_log:
      level: warn       # supports levels, lower to higher: debug, info, warn, error, panic, fatal
      file_path:
        logs/error.log  # supports relative path, absolute path, standard output
                        # such as: logs/error.log, /tmp/logs/error.log, /dev/stdout, /dev/stderr
authentication:
  secret:
    secret              # secret for jwt token generation.
                        # NOTE: Highly recommended to modify this value to protect `manager api`.
                        # if it's default value, when `manager api` start, it will generate a random string to replace it.
  expire_time: 3600     # jwt token expire time, in second
  users:
    - username: admin   # username and password for login `manager api`
      password: admin
    - username: user
      password: user

plugin_attr:
  prometheus:
    export_addr:
      ip: "0.0.0.0"
      port: 9091

plugins:
  - authz-casbin
  - jwt-extractor
  - jwt-auth

Here is my docker-compose.yml file found in example/docker-compose.yml

version: "3"

services:
  apisix:
    image: apache/apisix:${APISIX_IMAGE_TAG:-3.11.0-debian}
    restart: always
    volumes:
      - ./apisix_conf/config.yaml:/usr/local/apisix/conf/config.yaml:ro
      - ./apisix/plugins/custom-plugins:/usr/local/apisix/custom-plugin/apisix/plugins:ro
    depends_on:
      - etcd
    ##network_mode: host
    ports:
      - "9180:9180/tcp"
      - "9080:9080/tcp"
      - "9091:9091/tcp"
      - "9443:9443/tcp"
      - "9092:9092/tcp"
    networks:
      apisix:

  etcd:
    image: bitnami/etcd:3.5.11
    restart: always
    volumes:
      - etcd_data:/bitnami/etcd
    environment:
      ETCD_ENABLE_V2: "true"
      ALLOW_NONE_AUTHENTICATION: "yes"
      ETCD_ADVERTISE_CLIENT_URLS: "http://etcd:2379"
      ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379"
    ports:
      - "2379:2379/tcp"
    networks:
      apisix:

  web1:
    image: nginx:1.19.0-alpine
    restart: always
    volumes:
      - ./upstream/web1.conf:/etc/nginx/nginx.conf
    ports:
      - "9081:80/tcp"
    environment:
      - NGINX_PORT=80
    networks:
      apisix:

  web2:
    image: nginx:1.19.0-alpine
    restart: always
    volumes:
      - ./upstream/web2.conf:/etc/nginx/nginx.conf
    ports:
      - "9082:80/tcp"
    environment:
      - NGINX_PORT=80
    networks:
      apisix:

  prometheus:
    image: prom/prometheus:v2.25.0
    restart: always
    volumes:
      - ./prometheus_conf/prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - "9090:9090"
    networks:
      apisix:

  grafana:
    image: grafana/grafana:7.3.7
    restart: always
    ports:
      - "3000:3000"
    volumes:
      - "./grafana_conf/provisioning:/etc/grafana/provisioning"
      - "./grafana_conf/dashboards:/var/lib/grafana/dashboards"
      - "./grafana_conf/config/grafana.ini:/etc/grafana/grafana.ini"
    networks:
      apisix:

  dashboard:
    image: apache/apisix-dashboard
    restart: always
    ports:
      - "9000:9000"
    volumes:
      - ../all-in-one/apisix-dashboard/conf.yaml:/usr/local/apisix-dashboard/conf/conf.yaml
    networks:
      apisix:
    depends_on:
      - etcd

networks:
  apisix:
    driver: bridge

volumes:
  etcd_data:
    driver: local

The issue here is that I dont see logs in the /usr/local/apisix-dashboard/logs/error.log and /usr/local/apisix-dashboard/logs/access.log nor in /usr/local/apisix/logs/error.log.

I'm not sure what I'm doing wrongly. I can see the plugin files are properly copied into the apisix server container in the folder I described:

❯ docker exec docker-apisix-apisix-1 ls /usr/local/apisix/custom-plugin/apisix/plugins
authz-casbin.lua
jwt-extractor.lua

but when I make the following curl request

curl --request PUT \
  --url http://127.0.0.1:9180/apisix/admin/routes \
  --header 'Content-Type: application/json' \
  --header 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' \
  --data '{
  "id":"jwt-extractor",
  "uri":"/test",
  "plugins": {
        "jwt-extractor": {
            "role_claim": "role"
        }
    }
}'

I get the following response back

{
    "error_msg": "unknown plugin [jwt-extractor]"
}

Environment

  • APISIX version (run apisix version): 3.11.0-debian
  • Operating system (run uname -a): debian (docker image)
  • OpenResty / Nginx version (run openresty -V or nginx -V): 1.19,0
  • etcd version, if relevant (run curl http://127.0.0.1:9090/v1/server_info): 3.5.0 (the curl request returns 404)
  • APISIX Dashboard version, if relevant: 3.0.1
  • Plugin runner version, for issues related to plugin runners: Not Sure where to get this
  • LuaRocks version, for installation issues (run luarocks --version): Not Sure where to get this

I tried following the exact steps here with the same commands and all, and after trying this curl request

curl --request PUT \
  --url http://127.0.0.1:9180/apisix/admin/routes \
  --header 'Content-Type: application/json' \
  --header 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' \
  --data '{
    "id": "openapi-file-proxy",
    "uri": "/openapi.yaml",
    "plugins": {
        "file-proxy": {
            "path": "/usr/local/apisix/openapi.yaml"
        }
    }
}'

It still failed to find it. Below is the proof

{
    "error_msg": "unknown plugin [file-proxy]"
}

Solution

  • You're setting the plugin list for the dashboard but not for APISIX.

    You have to add the custom plugin in apisix_conf/config.yaml. Example:

    apisix:
      node_listen: 9080              # APISIX listening port
      enable_ipv6: false
      # Add these lines
      extra_lua_path: "/usr/local/apisix/custom-plugin/?.lua"
      log_level: debug
      enable_control: true
      control:
        ip: "0.0.0.0"
        port: 9092
      
    deployment:
      admin:
        allow_admin:               # https://nginx.org/en/docs/http/ngx_http_access_module.html#allow
          - 0.0.0.0/0              # We need to restrict ip access rules for security. 0.0.0.0/0 is for test.
    
        admin_key:
          - name: "admin"
            key: edd1c9f034335f136f87ad84b625c8f1
            role: admin                 # admin: manage all configuration data
    
          - name: "viewer"
            key: 4054f7cf07e344346cd3f287985e76a2
            role: viewer
    
      etcd:
        host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
          - "http://etcd:2379"          # multiple etcd address
        prefix: "/apisix"               # apisix configurations prefix
        timeout: 30                     # 30 seconds
    
    plugin_attr:
      prometheus:
        export_addr:
          ip: "0.0.0.0"
          port: 9091
    
    plugins:
      - authz-casbin
      - jwt-extractor
      - jwt-auth
    

    However, this will overwrite the default plugin list, disabling all the other plugins (you can find the default plugin list here: apache/apisix/conf/config.yaml.example)

    If you don't want it to be overwritten, you can take the full list and insert your custom plugins. Example:

    plugins:                           # plugin list (sorted by priority)
      - real-ip                        # priority: 23000
      - ai                             # priority: 22900
      - client-control                 # priority: 22000
      - proxy-control                  # priority: 21990
      - request-id                     # priority: 12015
      - zipkin                         # priority: 12011
      #- skywalking                    # priority: 12010
      #- opentelemetry                 # priority: 12009
      - ext-plugin-pre-req             # priority: 12000
      - fault-injection                # priority: 11000
      - mocking                        # priority: 10900
      - serverless-pre-function        # priority: 10000
      #- batch-requests                # priority: 4010
      - cors                           # priority: 4000
      - ip-restriction                 # priority: 3000
      - ua-restriction                 # priority: 2999
      - referer-restriction            # priority: 2990
      - csrf                           # priority: 2980
      - uri-blocker                    # priority: 2900
      - request-validation             # priority: 2800
      - chaitin-waf                    # priority: 2700
      - multi-auth                     # priority: 2600
      - openid-connect                 # priority: 2599
      - cas-auth                       # priority: 2597
      - authz-casbin                   # priority: 2560
      - authz-casdoor                  # priority: 2559
      - wolf-rbac                      # priority: 2555
      - ldap-auth                      # priority: 2540
      - hmac-auth                      # priority: 2530
      - basic-auth                     # priority: 2520
      - jwt-auth                       # priority: 2510
      - jwe-decrypt                    # priority: 2509
      - key-auth                       # priority: 2500
      - consumer-restriction           # priority: 2400
      - attach-consumer-label          # priority: 2399
      - forward-auth                   # priority: 2002
      - opa                            # priority: 2001
      - authz-keycloak                 # priority: 2000
      #- error-log-logger              # priority: 1091
      - proxy-cache                    # priority: 1085
      - body-transformer               # priority: 1080
      - ai-prompt-template             # priority: 1071
      - ai-prompt-decorator            # priority: 1070
      - ai-rag                         # priority: 1060
      - ai-content-moderation          # priority: 1040 TODO: compare priority with other ai plugins
      - proxy-mirror                   # priority: 1010
      - proxy-rewrite                  # priority: 1008
      - workflow                       # priority: 1006
      - api-breaker                    # priority: 1005
      - limit-conn                     # priority: 1003
      - limit-count                    # priority: 1002
      - limit-req                      # priority: 1001
      #- node-status                   # priority: 1000
      - ai-proxy                       # priority: 999
      #- brotli                        # priority: 996
      - gzip                           # priority: 995
      - server-info                    # priority: 990
      - traffic-split                  # priority: 966
      - redirect                       # priority: 900
      - response-rewrite               # priority: 899
      - degraphql                      # priority: 509
      - kafka-proxy                    # priority: 508
      #- dubbo-proxy                   # priority: 507
      - grpc-transcode                 # priority: 506
      - grpc-web                       # priority: 505
      - http-dubbo                     # priority: 504
      - public-api                     # priority: 501
      - prometheus                     # priority: 500
      - datadog                        # priority: 495
      - loki-logger                    # priority: 414
      - elasticsearch-logger           # priority: 413
      - echo                           # priority: 412
      - loggly                         # priority: 411
      - http-logger                    # priority: 410
      - splunk-hec-logging             # priority: 409
      - skywalking-logger              # priority: 408
      - google-cloud-logging           # priority: 407
      - sls-logger                     # priority: 406
      - tcp-logger                     # priority: 405
      - kafka-logger                   # priority: 403
      - rocketmq-logger                # priority: 402
      - syslog                         # priority: 401
      - udp-logger                     # priority: 400
      - file-logger                    # priority: 399
      - clickhouse-logger              # priority: 398
      - tencent-cloud-cls              # priority: 397
      - inspect                        # priority: 200
      #- log-rotate                    # priority: 100
      # <- recommend to use priority (0, 100) for your custom plugins
      - jwt-extractor
      
      - example-plugin                 # priority: 0
      #- gm                            # priority: -43
      #- ocsp-stapling                 # priority: -44
      - aws-lambda                     # priority: -1899
      - azure-functions                # priority: -1900
      - openwhisk                      # priority: -1901
      - openfunction                   # priority: -1902
      - serverless-post-function       # priority: -2000
      - ext-plugin-post-req            # priority: -3000
      - ext-plugin-post-resp           # priority: -4000
    

    Side Note

    You can set the execution priority for a specific plugin instance, without changing the plugin code, via its metadata. Example:

      "plugins": {
        "authz-casbin": {
          "_meta": {
            "priority": 2000
          }
        }
      }