How to create a paper.js Path.Circle that can be scaled/translated with affine matrix, but keep circle radius always fixed?

I have a paper.js layer where users can draw different paths (circles, line, etc). This layer can be panned or zoomed using mouse scrolling or dragging. I use affine matrix transformation to zoom/pan paths in this layer. This works rather well.

What i'm looking for is to create a circle (Path.Circle object) that can be panned and zoomed using matrix, just its radius has to be always fixed (5px for example). So basically matrix transformation needs to be applied only to position of circle, but not to outline of it.

Below is sample of a circle with radius 20px transformedPath, that is zoomed to 2x. Questions is how to keep radius of circle transformedPath fixed (radius = 20px), while applying the matrix transformation.

var transformedPath = new paper.Path.Circle(100,100,20);

transformedPath.strokeColor = 'black';

paper.project.activeLayer.matrix = new paper.Matrix(
  2, 0,
  0, 2,
  0, 0
);

UPDATE. Here's a more general sketch (code below) that is based on solution suggested by sasensi. In this sample blue circle radius stays fixed (this is correct), but problem is that blue circle also stays on the same place instead.

The desired outcome is that both circles move to new position, but blue circle radius stays fixed.

// draw a normal circle
var normalCircle = new Path.Circle({
    center: new Point(100,100),
    radius: 50,
    fillColor: 'orange',
});

// draw another circle that will have scale transformation reversed
var notScalingCircle = new Path.Circle({
    center: new Point(100,100),
    radius: 30,
    fillColor: 'blue',
});

// draw instructions
new PointText({
    content: 'press mouse button down to zoom in and see that blue circle size does not change',
    point: view.center + [0, -80],
    justification: 'center'
});

function transformLayer(matrix) {
    // scale layer
    // project.activeLayer.applyMatrix = false;
    project.activeLayer.matrix = matrix;

    // scale item with inverted amount to make it display like if it was not scaled with the layer
    notScalingCircle.matrix = matrix.clone().invert();
}

var matrix = new paper.Matrix(
        2,0,
        0,1.5,
        50,30
    );

// on mouse down...
function onMouseDown() {
    // ...scale up
    transformLayer(matrix);
}

// on mouse up...
function onMouseUp() {
    // ...scale down
    transformLayer(matrix.clone().invert());
}

Solution

I think that the best way do that is, when you scale your layer with a given amount, to scale your circle with the inverted amount.
That will make your circle look like if it was not scaled.
Here is a sketch demonstrating the solution:

// draw a normal circle
var normalCircle = new Path.Circle({
    center: view.center,
    radius: 50,
    fillColor: 'orange'
});

// draw another circle that will have scale transformation reversed
var notScalingCircle = new Path.Circle({
    center: view.center,
    radius: 30,
    fillColor: 'blue'
});

// draw instructions
new PointText({
    content: 'press mouse button down to zoom in and see that blue circle size does not change',
    point: view.center + [0, -80],
    justification: 'center'
});

function scaleLayer(amount) {
    // scale layer
    project.activeLayer.scale(amount, view.center);
    // scale item with inverted amount to make it display like if it was not scaled with the layer
    notScalingCircle.scale(1 / amount);
}

// on mouse down...
function onMouseDown() {
    // ...scale up
    scaleLayer(3);
}

// on mouse up...
function onMouseUp() {
    // ...scale down
    scaleLayer(1 / 3);
}

Edit

In response to the new example, you just have to invert the scaling transformation on the item and not all the matrix (which also include translation and rotation).
Here is the corrected sketch:

// draw a normal circle
var normalCircle = new Path.Circle({
    center: new Point(100, 100),
    radius: 50,
    fillColor: 'orange'
});

// draw another circle that will have scale transformation reversed
var notScalingCircle = new Path.Circle({
    center: new Point(100, 100),
    radius: 30,
    fillColor: 'blue'
});

// draw instructions
new PointText({
    content: 'press mouse button down to zoom in and see that blue circle size does not change',
    point: view.center + [0, -80],
    justification: 'center'
});

function transformLayer(matrix) {
    // scale layer
    // project.activeLayer.applyMatrix = false;
    project.activeLayer.matrix = matrix;

    // just invert the scale and not all matrix
    notScalingCircle.scale(1 / matrix.scaling.x, 1 / matrix.scaling.y);
}

var matrix = new paper.Matrix(
    2, 0,
    0, 1.5,
    50, 30
);

// on mouse down...
function onMouseDown() {
    // ...scale up
    transformLayer(matrix);
}

// on mouse up...
function onMouseUp() {
    // ...scale down
    transformLayer(matrix.clone().invert());
}